From cf492655e739465fbacc6c8ac2c903ff82e87f0e Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 16 Jan 2025 19:53:31 -0500 Subject: [PATCH 001/106] Added ring generation routine, position quick start to gensph, and modernized the gen_point API --- exputil/EXPmath.cc | 42 ++-- exputil/realize_model.cc | 514 +++++++++++++++++++++++++++++++-------- include/massmodel.H | 73 ++++-- utils/ICs/CMakeLists.txt | 4 +- utils/ICs/Disk2dHalo.cc | 4 +- utils/ICs/DiskHalo.cc | 7 +- utils/ICs/addring.cc | 192 +++++++++++++++ utils/ICs/gensph.cc | 73 +++++- 8 files changed, 734 insertions(+), 175 deletions(-) create mode 100644 utils/ICs/addring.cc diff --git a/exputil/EXPmath.cc b/exputil/EXPmath.cc index 74189d649..feed74357 100644 --- a/exputil/EXPmath.cc +++ b/exputil/EXPmath.cc @@ -61,12 +61,12 @@ namespace AltMath return ans; } -#define ACC 40.0 -#define BIGNO 1.0e10 -#define BIGNI 1.0e-10 - double bessj(int n,double x) { + const double ACC = 40.0; + const double BIGNO = 1.0e10; + const double BIGNI = 1.0e-10; + int j,jsum,m; double ax,bj,bjm,bjp,sum,tox,ans; @@ -114,10 +114,6 @@ namespace AltMath return x < 0.0 && n%2 == 1 ? -ans : ans; } -#undef ACC -#undef BIGNO -#undef BIGNI - double bessk0(double x) { double y,ans; @@ -174,12 +170,12 @@ namespace AltMath return ans; } -#define ACC 40.0 -#define BIGNO 1.0e10 -#define BIGNI 1.0e-10 - double bessi(int n,double x) { + const double ACC = 40.0; + const double BIGNO = 1.0e10; + const double BIGNI = 1.0e-10; + int j; double bi,bim,bip,tox,ans; double bessi0(double ); @@ -207,10 +203,6 @@ namespace AltMath } } -#undef ACC -#undef BIGNO -#undef BIGNI - double bessi0(double x) { double ax,ans; @@ -253,12 +245,12 @@ namespace AltMath return x < 0.0 ? -ans : ans; } -#define ACC 40.0 -#define BIGNO 1.0e10 -#define BIGNI 1.0e-10 - double jn_sph(int n, double x) { + const double ACC = 40.0; + const double BIGNO = 1.0e10; + const double BIGNI = 1.0e-10; + int j,m; double ax,bj,bjm,bjp,tox,ans; double jn_sph0(double x),jn_sph1(double x),jn_sphm1(double x); @@ -306,13 +298,10 @@ namespace AltMath return x < 0.0 && n%2 == 1 ? -ans : ans; } -#undef ACC -#undef BIGNO -#undef BIGNI - -#define TOL 1.0e-7 double jn_sph0(double x) { + const double TOL = 1.0e-7; + if (fabs(x)(get_min_radius(), gen_rmin); double Emax = get_pot(get_max_radius()); @@ -525,7 +517,9 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(int& ierr) if (Unit(random_gen)<0.5) vr *= -1.0; - azi = 2.0*M_PI*Unit(random_gen); + // Orientation of tangential velocity vector + // + double azi = 2.0*M_PI*Unit(random_gen); vt1 = vt*cos(azi); vt2 = vt*sin(azi); @@ -544,13 +538,10 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(int& ierr) << std::setw(12) << fmax << " %=" << std::setw(12) << static_cast(++toomany)/totcnt << std::endl; - ierr = 1; out.setZero(); - return out; + return {out, 1}; } - ierr = 0; - if (Unit(random_gen)>=0.5) vr *= -1.0; phi = 2.0*M_PI*Unit(random_gen); @@ -583,17 +574,139 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(int& ierr) #endif - return out; + return {out, 0}; } -Eigen::VectorXd AxiSymModel::gen_point_3d(double Emin, double Emax, - double Kmin, double Kmax, int& ierr) +AxiSymModel::PSret AxiSymModel::gen_point_3d(double r, double theta, double phi) { if (!dist_defined) { - std::cerr << "AxiSymModel: must define distribution before realizing!" - << std::endl; - exit (-1); + throw std::runtime_error("AxiSymModel: must define distribution before realizing!"); + } + + int it; // Iteration counter + double pot, vmax, vr=0.0, vt=0.0, eee, fmax; + double rmin = max(get_min_radius(), gen_rmin); + double Emax = get_pot(get_max_radius()); + + if (gen_firstime) { + + double tol = 1.0e-5; + double dx = (1.0 - 2.0*tol)/(numr-1); + double dy = (1.0 - 2.0*tol)/(numj-1); + + gen_mass.resize(gen_N); + gen_rloc.resize(gen_N); + gen_fmax.resize(gen_N); + +#ifdef DEBUG + std::cout << "gen_point_3d[" << ModelID << "]: " << rmin + << ", " << get_max_radius() << std::endl; +#endif + + double dr = (get_max_radius() - rmin)/gen_N; + + for (int i=0; ifmax ? zzz : fmax; + } + } + gen_fmax[i] = fmax*(1.0 + ftol); + + } + gen_firstime = false; + } + + fmax = odd2(r, gen_rloc, gen_fmax, 1); + pot = get_pot(r); + vmax = sqrt(2.0*fabs(Emax - pot)); + + // Trial velocity point + // + for (it=0; it distf(eee, r*vt)/fmax ) continue; + + if (Unit(random_gen)<0.5) vr *= -1.0; + + break; + } + + Eigen::VectorXd out(7); + + if (std::isnan(vr) or std::isnan(vt)) { + if (verbose) std::cout << "NaN found in AxiSymModel::gen_point_3d with r=" + << r << " theta=" << theta << " phi=" << phi + << std::endl; + out.setZero(); + return {out, 1}; + } + + static unsigned totcnt = 0, toomany = 0; + totcnt++; + + if (it==gen_itmax) { + if (verbose) + std::cerr << "Velocity selection failed [" << myid << "]: r=" + << std::setw(12) << r + << std::setw(12) << fmax + << " %=" << std::setw(12) + << static_cast(++toomany)/totcnt << std::endl; + out.setZero(); + return {out, 1}; + } + + double tv = 2.0*M_PI*Unit(random_gen); + double vth = vt*cos(tv); + double vph = vt*sin(tv); + + double cost = cos(theta); + double sint = sin(theta); + + double cosp = cos(phi); + double sinp = sin(phi); + + out[0] = 1.0; + out[1] = r*sint*cosp; + out[2] = r*sint*sinp; + out[3] = r*cost; + out[4] = vr*sint*cosp - vph*sinp + vth*cost*cosp; + out[5] = vr*sint*sinp + vph*cosp + vth*cost*sinp; + out[6] = vr*cost - vth*sint; + + return {out, 0}; +} + + +AxiSymModel::PSret AxiSymModel::gen_point_3d(double Emin, double Emax, + double Kmin, double Kmax) +{ + if (!dist_defined) { + throw std::runtime_error("AxiSymModel: must define distribution before realizing!"); } #ifdef DEBUG @@ -602,7 +715,7 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(double Emin, double Emax, #endif double r, vr, vt, vt1, vt2, E, K, J, jmax, w1t, eee, pot; - double phi, sint, cost, sinp, cosp, azi; + double phi, sint, cost, sinp, cosp; double rmin = max(get_min_radius(), gen_rmin); double rmax = get_max_radius(); @@ -688,7 +801,8 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(double Emin, double Emax, #endif } - // Enforce limits + // Enforce limits + // Emin = max(Emin, Emin_grid); Emin = min(Emin, Emax_grid); Emax = max(Emax, Emin_grid); @@ -733,9 +847,10 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(double Emin, double Emax, pot = get_pot(r); vt = J/r; - ierr = 0; - // Interpolation check (should be rare) - // Error condition is set + int ierr = 0; + + // Interpolation check (should be rare). Error condition is set. + // if (2.0*(E - pot) - vt*vt < 0.0) { ierr = 1; if (E < pot) E = pot; @@ -745,7 +860,9 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(double Emin, double Emax, if (Unit(random_gen)<0.5) vr *= -1.0; - azi = 2.0*M_PI*Unit(random_gen); + // Orientation of tangential velocity vector + // + double azi = 2.0*M_PI*Unit(random_gen); vt1 = vt*cos(azi); vt2 = vt*sin(azi); @@ -767,7 +884,7 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(double Emin, double Emax, #ifdef DEBUG - if (ierr) return out; + if (ierr) return {out, ierr}; eee = pot + 0.5*(out[4]*out[4]+out[5]*out[5]+out[6]*out[6]); orb.new_orbit(eee, 0.5); @@ -787,15 +904,15 @@ Eigen::VectorXd AxiSymModel::gen_point_3d(double Emin, double Emax, << std::endl; #endif - return out; + return {out, ierr}; } -Eigen::VectorXd AxiSymModel::gen_point_jeans_3d(int& ierr) +AxiSymModel::PSret AxiSymModel::gen_point_jeans_3d() { double r, d, vr, vt, vt1, vt2, vv, vtot; - double phi, sint, cost, sinp, cosp, azi; + double phi, sint, cost, sinp, cosp; double rmin = max(get_min_radius(), gen_rmin); if (gen_firstime_jeans) { @@ -838,8 +955,8 @@ Eigen::VectorXd AxiSymModel::gen_point_jeans_3d(int& ierr) // Debug - - ofstream test("test.grid"); + // + std::ofstream test("test.grid"); if (test) { test << "# [Jeans] Rmin=" << rmin @@ -885,7 +1002,9 @@ Eigen::VectorXd AxiSymModel::gen_point_jeans_3d(int& ierr) vr = vtot*xxx; vt = vtot*sqrt(yyy); - azi = 2.0*M_PI*Unit(random_gen); + // Orientation of tangential velocity vector + // + double azi = 2.0*M_PI*Unit(random_gen); vt1 = vt*cos(azi); vt2 = vt*sin(azi); @@ -907,24 +1026,20 @@ Eigen::VectorXd AxiSymModel::gen_point_jeans_3d(int& ierr) out[5] = vr * sint*sinp + vt1 * cost*sinp + vt2*cosp; out[6] = vr * cost - vt1 * sint; - ierr = 0; - - return out; + return {out, 0}; } -void AxiSymModel::gen_velocity(double* pos, double* vel, int& ierr) +int AxiSymModel::gen_velocity(double* pos, double* vel) { if (dof()!=3) bomb( "AxiSymModel: gen_velocity only implemented for 3d model!" ); - + if (!dist_defined) { - std::cerr << "AxiSymModel: must define distribution before realizing!" - << std::endl; - exit (-1); + throw std::runtime_error("AxiSymModel: must define distribution before realizing!"); } double r, pot, vmax, vr=0.0, vt, eee, vt1=0.0, vt2=0.0, fmax; - double phi, sint, cost, sinp, cosp, azi; + double phi, sint, cost, sinp, cosp; double rmin = max(get_min_radius(), gen_rmin); double Emax = get_pot(get_max_radius()); @@ -1021,7 +1136,9 @@ void AxiSymModel::gen_velocity(double* pos, double* vel, int& ierr) if (Unit(random_gen)<0.5) vr *= -1.0; - azi = 2.0*M_PI*Unit(random_gen); + // Orientation of tangential velocity vector + // + double azi = 2.0*M_PI*Unit(random_gen); vt1 = vt*cos(azi); vt2 = vt*sin(azi); @@ -1038,12 +1155,9 @@ void AxiSymModel::gen_velocity(double* pos, double* vel, int& ierr) << std::setw(12) << fmax << " %=" << std::setw(12) << static_cast(++toomany)/totcnt << std::endl; - ierr = 1; - return; + return 1; } - ierr = 0; - if (Unit(random_gen)>=0.5) vr *= -1.0; phi = atan2(pos[1], pos[0]); @@ -1055,18 +1169,19 @@ void AxiSymModel::gen_velocity(double* pos, double* vel, int& ierr) vel[0] = vr * sint*cosp + vt1 * cost*cosp - vt2*sinp; vel[1] = vr * sint*sinp + vt1 * cost*sinp + vt2*cosp; vel[2] = vr * cost - vt1 * sint; + + return 0; } -Eigen::VectorXd SphericalModelMulti::gen_point(int& ierr) +AxiSymModel::PSret SphericalModelMulti::gen_point() { if (!real->dist_defined || !fake->dist_defined) { - std::cerr << "SphericalModelMulti: input distribution functions must be defined before realizing!" << std::endl; - exit (-1); + throw std::runtime_error("SphericalModelMulti: input distribution functions must be defined before realizing!"); } double r, pot, vmax; double vr=0.0, vt=0.0, eee=0.0, vt1=0.0, vt2=0.0, fmax, emax; - double mass, phi, sint, cost, sinp, cosp, azi; + double mass, phi, sint, cost, sinp, cosp; double Emax = get_pot(get_max_radius()); @@ -1271,6 +1386,7 @@ Eigen::VectorXd SphericalModelMulti::gen_point(int& ierr) for (it=0; itget_mass(r); -Eigen::VectorXd - SphericalModelMulti::gen_point(double Emin, double Emax, double Kmin, - double Kmax, int& ierr) + pot = get_pot(r); + vmax = sqrt(2.0*fabs(Emax - pot)); + + fmax = 0.0; + for (int j=0; jdistf(eee, r*vt); + fmax = zzz>fmax ? zzz : fmax; + } + } + gen_fmax[i] = fmax*(1.0 + ftol); + + } + + // Debug + // + ofstream test("test_multi.grid"); + if (test) { + + test << "# Rmin=" << rmin_gen + << " Rmax=" << rmax_gen + << std::endl; + + for (int i=0; idistf(get_pot(exp(gen_rloc[i])), 0.5) + << std::endl; + } + } + + gen_firstime = false; + } + + r = max(rmin_gen, min(rmax_gen, radius)); + fmax = odd2(r, gen_rloc, gen_fmax, 1); + if (gen_logr) r = exp(r); + + pot = get_pot(r); + vmax = sqrt(2.0*max(Emax - pot, 0.0)); + + // Trial velocity point + // + // Diagnostic counters + int reject=0; + int it; // Iteration counter + for (it=0; it fake->distf(eee, r*vt)/fmax ) { + reject++; + continue; + } + + if (Unit(random_gen)<0.5) vr *= -1.0; + + // Orientation of tangential velocity vector + // + double azi = 2.0*M_PI*Unit(random_gen); + vt1 = vt*cos(azi); + vt2 = vt*sin(azi); + + break; + } + + Eigen::VectorXd out(7); + + if (it==gen_itmax) { + if (verbose) { + static unsigned totcnt = 0; + std::cerr << "Velocity selection failed [" << std::setw(7) << ++totcnt + << "," << std::setw(4) << myid << "]: r=" + << std::setw(12) << r + << std::setw(12) << fmax + << " reject=" << std::setw( 6) << reject + << std::endl; + } + out.setZero(); + return {out, 1}; + } + + if (Unit(random_gen)>=0.5) vr *= -1.0; + + double dfr = real->distf(eee, r*vt); + double dfn = fake->distf(eee, r*vt); + double rat = dfr/dfn; + + // Deep debug + // +#ifdef DEBUG + if (rat <= 0.0) { + std::cout << "[" << std::setw(3) << myid << "] Bad mass: rat=" + << std::setw(16) << rat + << " df(M)=" << std::setw(16) << dfr + << " df(N)=" << std::setw(16) << dfn + << " r=" << std::setw(16) << r + << " E=" << std::setw(16) << eee + << std::endl; + } +#endif + + double cost = cos(theta); + double sint = sin(theta); + double cosp = cos(phi); + double sinp = sin(phi); + + out[0] = rat; + out[1] = r * sint*cosp; + out[2] = r * sint*sinp; + out[3] = r * cost; + out[4] = vr * sint*cosp + vt1 * cost*cosp - vt2*sinp; + out[5] = vr * sint*sinp + vt1 * cost*sinp + vt2*cosp; + out[6] = vr * cost - vt1 * sint; + + int ierr = 0; + for (int i=0; i<7; i++) { + if (std::isnan(out[i]) || std::isinf(out[i])) ierr = 1; + } + + return {out, ierr}; +} + +AxiSymModel::PSret SphericalModelMulti::gen_point +(double Emin, double Emax, double Kmin, double Kmax) { if (!real->dist_defined || !fake->dist_defined) { - std::cerr << "SphericalModelMulti: input distribution functions must be defined before realizing!" << std::endl; - exit (-1); + throw std::runtime_error("SphericalModelMulti: input distribution functions must be defined before realizing!"); } double r, vr, vt, vt1, vt2, E, K, J, jmax, w1t, pot; - double phi, sint, cost, sinp, cosp, azi; + double phi, sint, cost, sinp, cosp; double rmin = max(get_min_radius(), gen_rmin); double rmax = get_max_radius(); @@ -1619,7 +1911,9 @@ Eigen::VectorXd vector wrvec; for (int j=0; j(Emin, Emin_grid); Emin = min(Emin, Emax_grid); Emax = max(Emax, Emin_grid); @@ -1714,9 +2009,10 @@ Eigen::VectorXd pot = get_pot(r); vt = J/r; - ierr = 0; - // Interpolation check (should be rare) - // Error condition is set + + // Interpolation check (should be rare). Error condition is set. + // + int ierr = 0; if (2.0*(E - pot) - vt*vt < 0.0) { ierr = 1; if (E < pot) E = pot; @@ -1726,7 +2022,9 @@ Eigen::VectorXd if (Unit(random_gen)<0.5) vr *= -1.0; - azi = 2.0*M_PI*Unit(random_gen); + // Orientation of the tangential velocity vector + // + double azi = 2.0*M_PI*Unit(random_gen); vt1 = vt*cos(azi); vt2 = vt*sin(azi); @@ -1750,7 +2048,7 @@ Eigen::VectorXd if (std::isnan(out[i]) || std::isinf(out[i])) ierr = 1; } - return out; + return {out, ierr}; } diff --git a/include/massmodel.H b/include/massmodel.H index 88939322d..a9a6a75a3 100644 --- a/include/massmodel.H +++ b/include/massmodel.H @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -111,6 +112,10 @@ public: class AxiSymModel : public MassModel, public std::enable_shared_from_this { +public: + + using PSret = std::tuple; + protected: //@{ //! Stuff for gen_point @@ -126,11 +131,12 @@ protected: double gen_fomax, gen_ecut; //@} - Eigen::VectorXd gen_point_2d(int& ierr); - Eigen::VectorXd gen_point_2d(double r, int& ierr); - Eigen::VectorXd gen_point_3d(int& ierr); - Eigen::VectorXd gen_point_3d(double Emin, double Emax, double Kmin, double Kmax, int& ierr); - Eigen::VectorXd gen_point_jeans_3d(int& ierr); + PSret gen_point_2d(); + PSret gen_point_2d(double r); + PSret gen_point_3d(); + PSret gen_point_3d(double Emin, double Emax, double Kmin, double Kmax); + PSret gen_point_jeans_3d(); + PSret gen_point_3d(double r, double theta, double phi); double Emin_grid, Emax_grid, dEgrid, dKgrid; vector Egrid, Kgrid, EgridMass; @@ -212,55 +218,73 @@ public: void set_Ecut(double cut) { gen_ecut = cut; } //! Generate a phase-space point - virtual Eigen::VectorXd gen_point(int& ierr) { + virtual PSret gen_point() + { if (dof()==2) - return gen_point_2d(ierr); + return gen_point_2d(); else if (dof()==3) - return gen_point_3d(ierr); + return gen_point_3d(); else bomb( "dof must be 2 or 3" ); - return Eigen::VectorXd(); + return {Eigen::VectorXd(), 1}; } //! Generate a phase-space point using Jeans' equations - virtual Eigen::VectorXd gen_point_jeans(int& ierr) { + virtual PSret gen_point_jeans() + { if (dof()==2) bomb( "AxiSymModel: gen_point_jeans_2d(ierr) not implemented!" ); else if (dof()==3) - return gen_point_jeans_3d(ierr); + return gen_point_jeans_3d(); else bomb( "dof must be 2 or 3" ); - return Eigen::VectorXd(); + return {Eigen::VectorXd(), 1}; } //! Generate a phase-space point at a particular radius - virtual Eigen::VectorXd gen_point(double r, int& ierr) { + virtual PSret gen_point(double r) { + if (dof()==2) + return gen_point_2d(r); + else if (dof()==3) + bomb( "AxiSymModel: gen_point(r) not implemented!" ); + else + bomb( "AxiSymModel: dof must be 2 or 3" ); + + return {Eigen::VectorXd(), 1}; + } + + //! Generate a phase-space point at a particular radius, theta, and phi + virtual PSret + gen_point(double r, double theta, double phi) + { if (dof()==2) - return gen_point_2d(r, ierr); + bomb( "AxiSymModel: gen_point(r, theta, phi) not implemented!" ); else if (dof()==3) - bomb( "AxiSymModel: gen_point(r, ierr) not implemented!" ); + return gen_point_3d(r, theta, phi); else bomb( "AxiSymModel: dof must be 2 or 3" ); - return Eigen::VectorXd(); + return {Eigen::VectorXd(), 1}; } //! Generate a phase-space point at a particular energy and angular momentum - virtual Eigen::VectorXd gen_point(double Emin, double Emax, double Kmin, double Kmax, int& ierr) { + virtual PSret + gen_point(double Emin, double Emax, double Kmin, double Kmax) + { if (dof()==2) - bomb( "AxiSymModel: gen_point(r, ierr) not implemented!" ); + bomb( "AxiSymModel: gen_point(r) not implemented!" ); else if (dof()==3) - return gen_point_3d(Emin, Emax, Kmin, Kmax, ierr); + return gen_point_3d(Emin, Emax, Kmin, Kmax); else bomb( "AxiSymModel: dof must be 2 or 3" ); - return Eigen::VectorXd(); + return {Eigen::VectorXd(), 1}; } //! Generate a the velocity variate from a position - virtual void gen_velocity(double *pos, double *vel, int& ierr); + virtual int gen_velocity(double *pos, double *vel); }; @@ -501,9 +525,10 @@ public: //@{ //! Overloaded to provide mass distribution from Real and Number distribution from Fake - Eigen::VectorXd gen_point(int& ierr); - Eigen::VectorXd gen_point(double r, int& ierr); - Eigen::VectorXd gen_point(double Emin, double Emax, double Kmin, double Kmax, int& ierr); + PSret gen_point(); + PSret gen_point(double r); + PSret gen_point(double Emin, double Emax, double Kmin, double Kmax); + PSret gen_point(double r, double theta, double phi); //@} diff --git a/utils/ICs/CMakeLists.txt b/utils/ICs/CMakeLists.txt index 0c89b9942..281695e52 100644 --- a/utils/ICs/CMakeLists.txt +++ b/utils/ICs/CMakeLists.txt @@ -2,7 +2,7 @@ set(bin_PROGRAMS shrinkics gensph gendisk gendisk2d gsphere pstmod empinfo empdump eofbasis eofcomp testcoefs testcoefs2 testdeval forcetest hdf52accel forcetest2 cylcache modelfit test2d addsphmod - cubeics zangics slabics) + cubeics zangics slabics addring) set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} @@ -88,6 +88,8 @@ add_executable(zangics ZangICs.cc) add_executable(slabics genslab.cc massmodel1d.cc) +add_executable(addring addring.cc) + foreach(program ${bin_PROGRAMS}) target_link_libraries(${program} ${common_LINKLIB}) target_include_directories(${program} PUBLIC ${common_INCLUDE}) diff --git a/utils/ICs/Disk2dHalo.cc b/utils/ICs/Disk2dHalo.cc index f923e4792..ad0e9a61b 100644 --- a/utils/ICs/Disk2dHalo.cc +++ b/utils/ICs/Disk2dHalo.cc @@ -401,7 +401,7 @@ void Disk2dHalo::set_halo(vector& phalo, int nhalo, int npart) for (int i=0; igen_point(ierr); + std::tie(ps, ierr) = multi->gen_point(); if (ierr) count1++; } while (ierr); @@ -2329,7 +2329,7 @@ void Disk2dHalo::set_vel_halo(vector& part) // Use Eddington if (DF && 0.5*(1.0+erf((r-R_DF)/DR_DF)) > rndU(gen)) { - halo2->gen_velocity(&p.pos[0], &p.vel[0], nok); + nok = halo2->gen_velocity(&p.pos[0], &p.vel[0]); if (nok) { std::cout << "gen_velocity failed: " diff --git a/utils/ICs/DiskHalo.cc b/utils/ICs/DiskHalo.cc index c64a75c6a..0ff393af6 100644 --- a/utils/ICs/DiskHalo.cc +++ b/utils/ICs/DiskHalo.cc @@ -411,7 +411,7 @@ void DiskHalo::set_halo(vector& phalo, int nhalo, int npart) for (int i=0; igen_point(ierr); + std::tie(ps, ierr) = multi->gen_point(); if (ierr) count1++; } while (ierr); @@ -631,7 +631,7 @@ set_halo_coordinates(vector& phalo, int nhalo, int npart) p.pos[2] = r*costh; do { - ps = halo2->gen_point(ierr); + std::tie(ps, ierr) = halo2->gen_point(); } while (ierr); massp1 += p.mass; @@ -2560,7 +2560,8 @@ void DiskHalo::set_vel_halo(vector& part) // Use Eddington if (DF && 0.5*(1.0+erf((r-R_DF)/DR_DF)) > rndU(gen)) { - halo2->gen_velocity(&p.pos[0], &p.vel[0], nok); + + nok = halo2->gen_velocity(&p.pos[0], &p.vel[0]); if (nok) { std::cout << "gen_velocity failed: " diff --git a/utils/ICs/addring.cc b/utils/ICs/addring.cc new file mode 100644 index 000000000..6e4cedafe --- /dev/null +++ b/utils/ICs/addring.cc @@ -0,0 +1,192 @@ +/* + Add a ring of particles to an existing realization +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#ifdef HAVE_OMP_H +#include +#endif + +// EXP classes +// +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // Library globals +#include // Command-line parsing +#include // Ini-style config + + // Local headers +#include +#include + +int +main(int ac, char **av) +{ + //==================== + // Inialize MPI stuff + //==================== + + local_init_mpi(ac, av); + + //==================== + // Begin opt parsing + //==================== + + std::string halofile; + int DIVERGE, N; + double DIVERGE_RFAC, mass, radius, width, vdisp; + std::string outfile, config; + + const std::string mesg("Make a model from the combination of two spherical models. E.g. a DM halo and a bulge.\n"); + + cxxopts::Options options(av[0], mesg); + + options.add_options() + ("h,help", "Print this help message") + ("s,spline", "Set interplation in SphericalModelTable to 'spline' (default: linear)") + ("T,template", "Write template options file with current and all default values") + ("c,config", "Parameter configuration file", + cxxopts::value(config)) + ("N,number", "Number of particles", + cxxopts::value(N)->default_value("100000")) + ("DIVERGE", "Cusp power-law density extrapolation (0 means off)", + cxxopts::value(DIVERGE)->default_value("0")) + ("DIVERGE_RFAC", "Exponent for inner cusp extrapolation", + cxxopts::value(DIVERGE_RFAC)->default_value("1.0")) + ("i,halofile", "Halo spherical model table", + cxxopts::value(halofile)->default_value("SLGridSph.one")) + ("o,outfile", "Output particle file", + cxxopts::value(outfile)->default_value("addring.bods")) + ("M,mass", "Mass of ring", + cxxopts::value(mass)->default_value("0.1")) + ("R,radius", "Radius of ring", + cxxopts::value(radius)->default_value("1.0")) + ("W,width", "Width of ring", + cxxopts::value(width)->default_value("0.1")) + ("V,disp", "Velocity dispersion", + cxxopts::value(vdisp)->default_value("0.1")) + ; + + cxxopts::ParseResult vm; + + try { + vm = options.parse(ac, av); + } catch (cxxopts::OptionException& e) { + if (myid==0) std::cout << "Option error: " << e.what() << std::endl; + MPI_Finalize(); + exit(-1); + } + + // Write YAML template config file and exit + // + if (vm.count("template")) { + // Write template file + // + if (myid==0) SaveConfig(vm, options, "template.yaml"); + + MPI_Finalize(); + return 0; + } + + // Print help message and exit + // + if (vm.count("help")) { + if (myid == 0) { + std::cout << options.help() << std::endl << std::endl + << "Examples: " << std::endl + << "\t" << "Use parameters read from a YAML config file" << std::endl + << "\t" << av[0] << " --config=addsphmod.config" << std::endl << std::endl + << "\t" << "Generate a template YAML config file from current defaults" << std::endl + << "\t" << av[0] << " --template=template.yaml" << std::endl << std::endl; + } + MPI_Finalize(); + return 0; + } + + // Read parameters fron the YAML config file + // + if (vm.count("config")) { + try { + vm = LoadConfig(options, config); + } catch (cxxopts::OptionException& e) { + if (myid==0) std::cout << "Option error in configuration file: " + << e.what() << std::endl; + MPI_Finalize(); + return 0; + } + } + + if (vm.count("spline")) { + SphericalModelTable::linear = 0; + } else { + SphericalModelTable::linear = 1; + } + + // Open output file + // + std::ofstream out(outfile); + if (not out) throw std::runtime_error("Cannot open output file " + outfile); + + // Model + // + SphericalModelTable model(halofile, DIVERGE, DIVERGE_RFAC); + + + // Random setup + // + std::random_device rd{}; + std::mt19937 gen{rd()}; + + // Unit normal distribution + // + std::normal_distribution<> d{0.0, 1.0}; + std::uniform_real_distribution<> u{0.0, 1.0}; + + + double pmass = mass/N; + Eigen::Vector3d pos, vel; + + for (int i=0; i(ELIMIT)->default_value("false")) ("VTEST", "Test gen_velocity() generation", cxxopts::value(VTEST)->default_value("false")) + ("Vquiet", "Number of angular points on a regular spherical grid (0 means no quiet)", + cxxopts::value(Vquiet)->default_value("0")) ("Emin0", "Minimum energy (if ELIMIT=true)", cxxopts::value(Emin0)->default_value("-3.0")) ("Emax0", "Maximum energy (if ELIMIT=true)", @@ -703,23 +705,72 @@ main(int argc, char **argv) std::vector PS; Eigen::VectorXd zz = Eigen::VectorXd::Zero(7); + std::vector rmass; + int nradial=0, nphi=0, ncost=0; + if (Vquiet) { + nphi = 2*Vquiet; + ncost = Vquiet; + nradial = floor(N/(nphi*ncost)); + if (nradial*nphi*ncost < N) nradial++; + rmass.resize(nradial); + + // Set up mass grid + double rmin = hmodel->get_min_radius(); + double rmax = hmodel->get_max_radius(); + double mmas = hmodel->get_mass(rmax); + double dmas = mmas/nradial; + for (int i=0; i double + { return (mas - hmodel->get_mass(r)); }; + rmass[i] = zbrent(loc, rmin, rmax, 1.0e-8); + } + } + for (int n=beg; ngen_point(Emin0, Emax0, Kmin0, Kmax0, ierr); + if (Vquiet) { + // which radial ring + int nr = floor(n/(nphi*ncost)); + int rm = n - nr*nphi*ncost; + // which elevational plane + int nt = rm/nphi; + // which azimuth + int np = rm - nt*nphi; + + assert((nr>=0 && nr=0 && nt=0 && npgen_point(r, acos(cost), phi); + + for (int i=0; i<3; i++) { + if (std::isnan(ps[i+3])) { + std::cout << "Vel NaN with r=" << r << " cost=" << cost << " phi=" << phi << std::endl; + } + } + } + else if (ELIMIT) + std::tie(ps, ierr) = rmodel->gen_point(Emin0, Emax0, Kmin0, Kmax0); else if (VTEST) { - ps = rmodel->gen_point(ierr); - rmodel->gen_velocity(&ps[1], &ps[4], ierr); - if (ierr) { - std::cout << "gen_velocity failed: " - << ps[0] << " " - << ps[1] << " " - << ps[2] << "\n"; + std::tie(ps, ierr) = rmodel->gen_point(); + if (ierr==0) { + ierr = rmodel->gen_velocity(&ps[1], &ps[4]); + if (ierr) { + std::cout << "gen_velocity failed: " + << ps[0] << " " + << ps[1] << " " + << ps[2] << "\n"; + } } } else - ps = rmodel->gen_point(ierr); + std::tie(ps, ierr) = rmodel->gen_point(); if (ierr) count++; } while (ierr); From 51298caccf2a586d1a91b38c527578ddb6b28cf4 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Fri, 17 Jan 2025 14:19:06 -0500 Subject: [PATCH 002/106] Update particle numbers to exactly fill each radial shell --- utils/ICs/gensph.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/utils/ICs/gensph.cc b/utils/ICs/gensph.cc index d885f1282..8bfe6ca5f 100644 --- a/utils/ICs/gensph.cc +++ b/utils/ICs/gensph.cc @@ -522,6 +522,16 @@ main(int argc, char **argv) } + int nradial=0, nphi=0, ncost=0; + if (Vquiet) { + nphi = 2*Vquiet; + ncost = Vquiet; + nradial = floor(N/(nphi*ncost)); + + // Reset N + N = nradial*nphi*ncost; + } + double mass = hmodel->get_mass(hmodel->get_max_radius())/N; AxiSymModPtr rmodel = hmodel; @@ -706,12 +716,7 @@ main(int argc, char **argv) Eigen::VectorXd zz = Eigen::VectorXd::Zero(7); std::vector rmass; - int nradial=0, nphi=0, ncost=0; if (Vquiet) { - nphi = 2*Vquiet; - ncost = Vquiet; - nradial = floor(N/(nphi*ncost)); - if (nradial*nphi*ncost < N) nradial++; rmass.resize(nradial); // Set up mass grid From 37cbf888de7ca9cdfba4b75a908c8667ef7efa0c Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Fri, 17 Jan 2025 14:22:14 -0500 Subject: [PATCH 003/106] Stagger radial bins from mass end points; add diagnostic mass ring profile --- exputil/realize_model.cc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/exputil/realize_model.cc b/exputil/realize_model.cc index 6cb6c6ffa..4b7c4ee13 100644 --- a/exputil/realize_model.cc +++ b/exputil/realize_model.cc @@ -607,7 +607,7 @@ AxiSymModel::PSret AxiSymModel::gen_point_3d(double r, double theta, double phi) double dr = (get_max_radius() - rmin)/gen_N; for (int i=0; i Date: Sat, 18 Jan 2025 12:50:50 -0500 Subject: [PATCH 004/106] Added Fibonacci spiral mapped to the sphere for uniform coverage of shells --- utils/ICs/gensph.cc | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/utils/ICs/gensph.cc b/utils/ICs/gensph.cc index 8bfe6ca5f..ea2b8bcc9 100644 --- a/utils/ICs/gensph.cc +++ b/utils/ICs/gensph.cc @@ -72,7 +72,7 @@ main(int argc, char **argv) double X0, Y0, Z0, U0, V0, W0, TOLE; double Emin0, Emax0, Kmin0, Kmax0, RBAR, MBAR, BRATIO, CRATIO, SMOOTH; bool LOGR, ELIMIT, VERBOSE, GRIDPOT, MODELS, EBAR, zeropos, zerovel; - bool VTEST; + bool VTEST, FIB=false; std::string INFILE, MMFILE, OUTFILE, OUTPS, config; #ifdef DEBUG @@ -94,6 +94,7 @@ main(int argc, char **argv) options.add_options() ("h,help", "Print this help message") ("v,verbose", "Print additional diagnostic information") + ("f,Fibonacci", "Use a Fibonacci lattice to tile the spherical shells") ("T,template", "Write template options file with current and all default values") ("c,config", "Parameter configuration file", cxxopts::value(config)) @@ -256,6 +257,8 @@ main(int argc, char **argv) if (vm.count("verbose")) VERBOSE = true; else VERBOSE = false; + if (vm.count("Fibonacci")) FIB = true; + // Prepare output streams and create new files // std::ostringstream sout; @@ -524,11 +527,12 @@ main(int argc, char **argv) int nradial=0, nphi=0, ncost=0; if (Vquiet) { - nphi = 2*Vquiet; - ncost = Vquiet; + nphi = 2*Vquiet; // # of aximuthal angles + ncost = Vquiet; // # of colatitude angles + // # of radial shells nradial = floor(N/(nphi*ncost)); - // Reset N + // Reset N to fill all spherical shells N = nradial*nphi*ncost; } @@ -736,21 +740,28 @@ main(int argc, char **argv) do { if (Vquiet) { - // which radial ring + // Which radial ring? int nr = floor(n/(nphi*ncost)); int rm = n - nr*nphi*ncost; - // which elevational plane + // Which elevational plane? int nt = rm/nphi; - // which azimuth + // Which azimuth? int np = rm - nt*nphi; assert((nr>=0 && nr=0 && nt=0 && npgen_point(r, acos(cost), phi); From 846e4f4292428b9ddde2e853cc5105a8c9c65e24 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 18 Jan 2025 12:51:16 -0500 Subject: [PATCH 005/106] Fixed error in velocity assignment at fixed radius --- exputil/realize_model.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/exputil/realize_model.cc b/exputil/realize_model.cc index 4b7c4ee13..df43b69fc 100644 --- a/exputil/realize_model.cc +++ b/exputil/realize_model.cc @@ -655,10 +655,11 @@ AxiSymModel::PSret AxiSymModel::gen_point_3d(double r, double theta, double phi) for (it=0; it distf(eee, r*vt)/fmax ) continue; From 87666df9c24af38ce8373304d19f72364ce3676b Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 21 Jan 2025 15:20:25 -0500 Subject: [PATCH 006/106] Updates to gensph for quiet start variations --- exputil/massmodel_dist.cc | 40 +++-- exputil/realize_model.cc | 158 +++++++++++++++++-- include/massmodel.H | 21 ++- utils/ICs/gensph.cc | 319 ++++++++++++++++++++++++++++---------- 4 files changed, 434 insertions(+), 104 deletions(-) diff --git a/exputil/massmodel_dist.cc b/exputil/massmodel_dist.cc index 27e2d877f..618a4bd8e 100644 --- a/exputil/massmodel_dist.cc +++ b/exputil/massmodel_dist.cc @@ -49,14 +49,14 @@ #include #include -#define OFFSET 1.0e-3 -#define OFFTOL 1.2 +const double OFFSET=1.0e-3; +const double OFFTOL=1.2; extern double gint_0(double a, double b, std::function f, int NGauss); extern double gint_2(double a, double b, std::function f, int NGauss); -#define TSTEP 1.0e-8 -#define NGauss 96 +const double TSTEP=1.0e-8; +const int NGauss=96; static int DIVERGE=0; @@ -109,6 +109,10 @@ void SphericalModelTable::setup_df(int NUM, double RA) rhoQy.resize(num); rhoQy2.resize(num); + rhoQx. setZero(); + rhoQy. setZero(); + rhoQy2.setZero(); + for (int i=0; i::max(); + // foffset = -std::numeric_limits::max(); + foffset = -1.0e42; dfc.Q[dfc.num-1] = Qmax; dfc.ffQ[dfc.num-1] = 0.0; fac = 1.0/(sqrt(8.0)*M_PI*M_PI); @@ -184,6 +194,13 @@ void SphericalModelTable::setup_df(int NUM, double RA) df.ffQ .resize(NUM); df.fQ2 .resize(NUM); df.ffQ2.resize(NUM); + + df.Q .setZero(); + df.fQ .setZero(); + df.ffQ .setZero(); + df.fQ2 .setZero(); + df.ffQ2.setZero(); + df.num = NUM; df.ra2 = ra2; @@ -194,7 +211,8 @@ void SphericalModelTable::setup_df(int NUM, double RA) df.Q[df.num-1] = Qmax; df.ffQ[df.num-1] = 0.0; fac = 1.0/(sqrt(8.0)*M_PI*M_PI); - foffset = -std::numeric_limits::max(); + // foffset = -std::numeric_limits::max(); + foffset = -1.0e42; for (int i=df.num-2; i>=0; i--) { df.Q[i] = df.Q[i+1] - dQ; Q = df.Q[i]; @@ -233,7 +251,7 @@ void SphericalModelTable::setup_df(int NUM, double RA) dist_defined = true; - debug_fdist(); + // debug_fdist(); } @@ -294,7 +312,7 @@ double SphericalModelTable::distf(double E, double L) if (!dist_defined) bomb("distribution function not defined"); - double d, g; + double d=0.0, g=0.0; if (chebyN) { @@ -341,7 +359,7 @@ double SphericalModelTable::dfde(double E, double L) if (!dist_defined) bomb("distribution function not defined"); - double d, g, h, d1; + double d=0, g=0, h=0, d1=0; if (chebyN) { @@ -399,7 +417,7 @@ double SphericalModelTable::dfdl(double E, double L) if (!dist_defined) bomb("distribution function not defined"); - double d, g, h, d1; + double d=0, g=0, h=0, d1=0; if (chebyN) { @@ -451,7 +469,7 @@ double SphericalModelTable::d2fde2(double E, double L) { if (!dist_defined) bomb("distribution function not defined"); - double d, g, h, k, d2; + double d=0, g=0, h=0, k=0, d2=0; if (chebyN) { diff --git a/exputil/realize_model.cc b/exputil/realize_model.cc index df43b69fc..3b2361adb 100644 --- a/exputil/realize_model.cc +++ b/exputil/realize_model.cc @@ -37,6 +37,7 @@ #include #include #include +#include #ifdef DEBUG #include @@ -107,9 +108,9 @@ AxiSymModel::PSret AxiSymModel::gen_point_2d() gen_orb.new_orbit(xxx, yyy); - double zzz = distf(xxx, gen_orb.AngMom())*gen_orb.Jmax() - /gen_orb.get_freq(1); - gen_fomax = zzz>gen_fomax ? zzz : gen_fomax; + double zzz = distf(xxx, gen_orb.AngMom())*gen_orb.Jmax()/gen_orb.get_freq(1); + + if (zzz>gen_fomax) gen_fomax = zzz; } } @@ -203,7 +204,7 @@ AxiSymModel::PSret AxiSymModel::gen_point_2d() eee = pot + 0.5*(vr*vr + vt*vt); double zzz = distf(eee, gen_rloc[i]*vt); - fmax = zzz>fmax ? zzz : fmax; + if (zzz > fmax) fmax = zzz; } } gen_fmax[i] = fmax*(1.0 + ftol); @@ -232,7 +233,7 @@ AxiSymModel::PSret AxiSymModel::gen_point_2d() if (Unit(random_gen) > distf(eee, r*vt)/fmax ) continue; - if (Unit(random_gen)<0.5) vr *= -1.0; + if (Unit(random_gen) < 0.5) vr *= -1.0; phi = 2.0*M_PI*Unit(random_gen); @@ -321,7 +322,7 @@ AxiSymModel::PSret AxiSymModel::gen_point_2d(double r) eee = pot + 0.5*(vr*vr + vt*vt); double zzz = distf(eee, gen_rloc[i]*vt); - fmax = zzz>fmax ? zzz : fmax; + if (zzz>fmax) fmax = zzz; } } gen_fmax[i] = fmax*(1.0 + ftol); @@ -347,7 +348,7 @@ AxiSymModel::PSret AxiSymModel::gen_point_2d(double r) if (Unit(random_gen) > distf(eee, r*vt)/fmax ) continue; - if (Unit(random_gen)<0.5) vr *= -1.0; + if (Unit(random_gen) < 0.5) vr *= -1.0; phi = 2.0*M_PI*Unit(random_gen); @@ -453,7 +454,7 @@ AxiSymModel::PSret AxiSymModel::gen_point_3d() eee = pot + 0.5*(vr*vr + vt*vt); double zzz = distf(eee, r*vt); - fmax = zzz>fmax ? zzz : fmax; + if (zzz>fmax) fmax = zzz; } } gen_fmax[i] = fmax*(1.0 + ftol); @@ -625,7 +626,7 @@ AxiSymModel::PSret AxiSymModel::gen_point_3d(double r, double theta, double phi) eee = pot + 0.5*(vr*vr + vt*vt); double zzz = distf(eee, gen_rloc[i]*vt); - fmax = zzz>fmax ? zzz : fmax; + if (zzz>fmax) fmax = zzz; } } gen_fmax[i] = fmax*(1.0 + ftol); @@ -714,6 +715,137 @@ AxiSymModel::PSret AxiSymModel::gen_point_3d(double r, double theta, double phi) return {out, 0}; } +AxiSymModel::PSret AxiSymModel::gen_point_3d_iso +(double r, double theta, double phi, int N, int nv, int na, + double PHI, double THETA, double PSI) +{ + if (!dist_defined) { + throw std::runtime_error("AxiSymModel: must define distribution before realizing!"); + } + + double Emax = get_pot(get_max_radius()); + double vpot = get_pot(r); + double v3 = pow(2.0*fabs(Emax - vpot), 1.5); + int knots = 20; + + // Sanity check + assert((N < nv*na)); + + // Rotation matrix + static Eigen::Matrix3d rotate; + + // Assume isotropic and generate new r slice + if (fabs(gen_lastr-r) > 1.0e-14) { + gen_mass.resize(gen_N+1); + gen_rloc.resize(gen_N+1); + gen_lastr = r; + +#ifdef DEBUG + std::cout << "gen_point_3d_iso[" << ModelID << "]: " << rmin + << ", " << get_max_radius() << std::endl; +#endif + + LegeQuad lv(knots); + + double dv3 = v3/gen_N; + + gen_rloc[0] = 0.0; + gen_mass[0] = 0.0; + for (int i=0; ifmax ? zzz : fmax; + if (zzz>fmax) fmax = zzz; } } gen_fmax[i] = fmax*(1.0 + ftol); @@ -1567,7 +1699,8 @@ AxiSymModel::PSret SphericalModelMulti::gen_point(double radius) eee = pot + 0.5*(vr*vr + vt*vt); double zzz = fake->distf(eee, r*vt); - fmax = zzz>fmax ? zzz : fmax; + + if (zzz>fmax) fmax = zzz; } } gen_fmax[i] = fmax*(1.0 + ftol); @@ -1750,7 +1883,8 @@ SphericalModelMulti::gen_point(double radius, double theta, double phi) eee = pot + 0.5*(vr*vr + vt*vt); double zzz = fake->distf(eee, r*vt); - fmax = zzz>fmax ? zzz : fmax; + + if (zzz>fmax) fmax = zzz; } } gen_fmax[i] = fmax*(1.0 + ftol); diff --git a/include/massmodel.H b/include/massmodel.H index a9a6a75a3..233e2f257 100644 --- a/include/massmodel.H +++ b/include/massmodel.H @@ -128,7 +128,7 @@ protected: bool gen_firstime_jeans; Eigen::VectorXd gen_rloc, gen_mass, gen_fmax; SphericalOrbit gen_orb; - double gen_fomax, gen_ecut; + double gen_fomax, gen_ecut, gen_lastr=-1.0; //@} PSret gen_point_2d(); @@ -137,6 +137,9 @@ protected: PSret gen_point_3d(double Emin, double Emax, double Kmin, double Kmax); PSret gen_point_jeans_3d(); PSret gen_point_3d(double r, double theta, double phi); + PSret gen_point_3d_iso(double r, double theta, double phi, + int N, int nv, int na, + double PHI=0, double THETA=0, double PSI=0); double Emin_grid, Emax_grid, dEgrid, dKgrid; vector Egrid, Kgrid, EgridMass; @@ -269,6 +272,22 @@ public: return {Eigen::VectorXd(), 1}; } + //! Generate a phase-space point at a particular radius, theta, and phi + virtual PSret + gen_point_iso(double r, double theta, double phi, int N, int nv, int na, + double PHI, double THETA, double PSI) + { + if (dof()==2) + bomb( "AxiSymModel: gen_point(r, theta, phi) not implemented!" ); + else if (dof()==3) + return gen_point_3d_iso(r, theta, phi, N, nv, na, + PHI, THETA, PSI); + else + bomb( "AxiSymModel: dof must be 2 or 3" ); + + return {Eigen::VectorXd(), 1}; + } + //! Generate a phase-space point at a particular energy and angular momentum virtual PSret gen_point(double Emin, double Emax, double Kmin, double Kmax) diff --git a/utils/ICs/gensph.cc b/utils/ICs/gensph.cc index ea2b8bcc9..9541ee21f 100644 --- a/utils/ICs/gensph.cc +++ b/utils/ICs/gensph.cc @@ -61,18 +61,21 @@ using namespace __EXP__; #include #endif +#include + // Global variables int main(int argc, char **argv) { int HMODEL, N, NUMDF, NUMR, NUMJ, NUME, NUMG, NREPORT, SEED, ITMAX; - int NUMMODEL, RNUM, DIVERGE, DIVERGE2, LINEAR, NUMINT, NI, ND, Vquiet; + int NUMMODEL, RNUM, DIVERGE, DIVERGE2, LINEAR, NUMINT, NI, ND, Nangle; + int Nrepl=1, Nfib=1; double DIVERGE_RFAC, DIVERGE_RFAC2, NN, MM, RA, RMODMIN, RMOD, EPS; double X0, Y0, Z0, U0, V0, W0, TOLE; double Emin0, Emax0, Kmin0, Kmax0, RBAR, MBAR, BRATIO, CRATIO, SMOOTH; bool LOGR, ELIMIT, VERBOSE, GRIDPOT, MODELS, EBAR, zeropos, zerovel; - bool VTEST, FIB=false; + bool VTEST; std::string INFILE, MMFILE, OUTFILE, OUTPS, config; #ifdef DEBUG @@ -94,7 +97,6 @@ main(int argc, char **argv) options.add_options() ("h,help", "Print this help message") ("v,verbose", "Print additional diagnostic information") - ("f,Fibonacci", "Use a Fibonacci lattice to tile the spherical shells") ("T,template", "Write template options file with current and all default values") ("c,config", "Parameter configuration file", cxxopts::value(config)) @@ -180,8 +182,12 @@ main(int argc, char **argv) cxxopts::value(ELIMIT)->default_value("false")) ("VTEST", "Test gen_velocity() generation", cxxopts::value(VTEST)->default_value("false")) - ("Vquiet", "Number of angular points on a regular spherical grid (0 means no quiet)", - cxxopts::value(Vquiet)->default_value("0")) + ("Nangle", "Number of angular points on a regular spherical grid (0 skips this algorithm). Nangle>0 takes precidence over other algorithms.", + cxxopts::value(Nangle)->default_value("0")) + ("Nrepl", "Number of replicates in orbital plane (1 skips the Sellwood 1997 algorithm)", + cxxopts::value(Nrepl)->default_value("1")) + ("Nfib", "Number of points on the sphere for each orbit. Replicate the orbits by tiling the angular momentum direction on the sphere using a Fibonnaci sequence. Default value of 1 implies one plane per orbit.", + cxxopts::value(Nfib)->default_value("1")) ("Emin0", "Minimum energy (if ELIMIT=true)", cxxopts::value(Emin0)->default_value("-3.0")) ("Emax0", "Maximum energy (if ELIMIT=true)", @@ -257,7 +263,15 @@ main(int argc, char **argv) if (vm.count("verbose")) VERBOSE = true; else VERBOSE = false; - if (vm.count("Fibonacci")) FIB = true; + // Use Fibonnaci for space-filling grid points + // + if (Nangle>0) { + Nfib = Nrepl = 1; + } + else { + Nfib = std::max(1, Nfib ); + Nrepl = std::max(1, Nrepl); + } // Prepare output streams and create new files // @@ -525,15 +539,34 @@ main(int argc, char **argv) } - int nradial=0, nphi=0, ncost=0; - if (Vquiet) { - nphi = 2*Vquiet; // # of aximuthal angles - ncost = Vquiet; // # of colatitude angles - // # of radial shells - nradial = floor(N/(nphi*ncost)); + int nfib=0, nangle=0, nshell=0; + if (Nangle) { + nangle = Nangle; + nfib = floor(sqrt(double(N)/(nangle*nangle))); + nshell = nfib*nangle; - // Reset N to fill all spherical shells - N = nradial*nphi*ncost; + // Shells in radius and velocity + // + N = nshell*nshell; + if (myid==0) + std::cout << std::setw(60) << std::setfill('-') << '-' + << std::setfill(' ') << std::endl + << "Fibonacci: N=" << N << " Nangle=" << Nangle + << " Nmag=" << nfib << " Nshell=" << nshell<< std::endl; + } + + // For replication algorithm; Nrepl=1 is the standard algorithm. + // + Nrepl *= Nfib; + int nplane = N/Nrepl; + N = Nrepl * nplane; + NREPORT = std::max(1, NREPORT/Nrepl); + + if (Nrepl > 1 and myid==0) { + std::cout << std::setw(60) << std::setfill('-') << '-' + << std::setfill(' ') << std::endl + << "Replication: N=" << N << " Nrepl=" << Nrepl/Nfib + << " Nfib=" << Nfib << " Norbits=" << nplane << std::endl; } double mass = hmodel->get_mass(hmodel->get_max_radius())/N; @@ -707,28 +740,29 @@ main(int argc, char **argv) double TT=0.0, WW=0.0, VC=0.0; if (myid==0) - std::cout << "-----------" << std::endl + std::cout << std::setw(60) << std::setfill('-') << '-' << std::endl + << std::setfill('-') << "Body count:" << std::endl; - int npernode = N/numprocs; + int npernode = nplane/numprocs; int beg = myid*npernode; int end = beg + npernode; - if (myid==numprocs-1) end = N; + if (myid==numprocs-1) end = nplane; std::vector PS; Eigen::VectorXd zz = Eigen::VectorXd::Zero(7); std::vector rmass; - if (Vquiet) { - rmass.resize(nradial); + if (Nangle) { + rmass.resize(nfib); // Set up mass grid double rmin = hmodel->get_min_radius(); double rmax = hmodel->get_max_radius(); double mmas = hmodel->get_mass(rmax); - double dmas = mmas/nradial; - for (int i=0; i double { return (mas - hmodel->get_mass(r)); }; @@ -736,34 +770,38 @@ main(int argc, char **argv) } } + constexpr double goldenRatio = 0.5*(1.0 + sqrt(5.0)); + for (int n=beg; n=0 && nr=0 && nt=0 && np=0 && nspace=0 && nveloc=0 && rmgen_point(r, acos(cost), phi); + double PHI = 2.0*M_PI * nr / goldenRatio; + double COST = 1.0 - 2.0 * nr / nangle; + double THETA = acos(COST) + 0.5*M_PI; + + // Get velocity coorindates + std::tie(ps, ierr) = rmodel->gen_point_iso(r, acos(cost), phi, + nveloc, nfib, nangle, + PHI, THETA, PHI); for (int i=0; i<3; i++) { if (std::isnan(ps[i+3])) { @@ -792,39 +830,156 @@ main(int argc, char **argv) if (ps[0] <= 0.0) negms++; - double RR=0.0; - for (int i=1; i<=3; i++) { - RR += ps[i]*ps[i]; - TT += 0.5*mass*ps[0]*ps[3+i]*ps[3+i]; - } - RR = sqrt(RR); - if (RR>=rmin) { - VC += -mass*ps[0]*rmodel->get_mass(RR)/RR; - WW += 0.5*mass*ps[0]*rmodel->get_pot(RR); - } + Eigen::Vector3d pos(ps[1], ps[2], ps[3]), pos1; + Eigen::Vector3d vel(ps[4], ps[5], ps[6]), vel1; + Eigen::Matrix3d proj, Iprj, rot = Eigen::Matrix3d::Identity(); + Eigen::Matrix3d projM, IprjM; - if (zeropos or zerovel) { - ps[0] *= mass; - PS.push_back(ps); - zz[0] += ps[0]; - if (zeropos) for (int i=1; i<3; i++) zz[i] -= ps[0]*ps[i]; - if (zerovel) for (int i=4; i<7; i++) zz[i] -= ps[0]*ps[i]; + double dq = 2.0*M_PI * Nfib / Nrepl, mfac = ps[0]; + + // Compute orbital plane basis + // + if (Nrepl>1) { + auto L = pos.cross(vel); // The angular momentum vector + if (pos.norm() < 1.0e-10 or L.norm() < 1.0e-10) { + proj = Iprj = Eigen::Matrix3d::Identity(); + } else { + auto X = pos/pos.norm(); // Radial unit vector, X. + auto Y = L.cross(X); // Choose a right-hand3d coordinate + Y = Y/Y.norm(); // system perpendicular to X and L. + auto Z = L/L.norm(); // Unit vector in the ang. mom. direction. + + // Azimuth of vertical plane containing L + double Phi = atan2(Z(1), Z(0)) + 0.5*M_PI; + + // Tilt of plane w.r.t to polar axis + double Theta = acos(Z(2)); + + // Location of X in orbital plane w.r.t line of nodes + Eigen::Vector3d lon(cos(Phi), sin(Phi), 0.0); + double dotp = lon.dot(X); + double Psi = acos(dotp); + auto dir = lon.cross(X); + + proj.row(0) = X; // Project to orbital plane frame + proj.row(1) = Y; + proj.row(2) = Z; + + Iprj.col(0) = X; // Project from orbital plane to original + Iprj.col(1) = Y; + Iprj.col(2) = Z; + + if (Nfib>1) { + // Quadrant geometry + int sgn1 = 1, sgn2 = 1; + if (Theta > 0.5*M_PI) sgn1 = -1; + if (dir(2) < 0.0) sgn2 = -1; + + // Sanity check for debugging + if (true) { + double norm = 0.0; + if (sgn1*sgn2==1) + norm = (proj - return_euler(Phi, Theta, Psi, 0)).norm(); + else + norm = (proj - return_euler(Phi, Theta, -Psi, 0)).norm(); + + if (fabs(norm) > 1.0e-10) { + std::cout << "Theta=" << Theta + << " dir(2)=" << dir(2) << std::endl; + } + } + + if (sgn1*sgn2==1) { + projM = return_euler(Phi, -Theta, Psi, 0); + IprjM = return_euler(Phi, -Theta, Psi, 1); + } else { + projM = return_euler(Phi, -Theta, -Psi, 0); + IprjM = return_euler(Phi, -Theta, -Psi, 1); + } + } + } } - else { - out << std::setw(20) << mass * ps[0]; - for (int i=1; i<=6; i++) out << std::setw(20) << ps[i]+ps0[i]; - if (NI) { - for (int n=0; n=rmin) { + VC += -mass*ps[0]*rmodel->get_mass(RR)/RR; + WW += 0.5*mass*ps[0]*rmodel->get_pot(RR); } - out << std::endl; + if (zeropos or zerovel) { + ps[0] *= mass; + PS.push_back(ps); + zz[0] += ps[0]; + if (zeropos) for (int i=1; i<3; i++) zz[i] -= ps[0]*ps[i]; + if (zerovel) for (int i=4; i<7; i++) zz[i] -= ps[0]*ps[i]; + } + else { + out << std::setw(20) << mass * ps[0]; + for (int i=1; i<=6; i++) out << std::setw(20) << ps[i]+ps0[i]; + + if (NI) { + for (int n=0; n1) { + + Eigen::Matrix3d invt; + double Q = dq*(q/Nfib+1), phi=0.0, cost=0.0; + int nfib = q % Nfib; + + if (Nfib>1) { + phi = 2.0*M_PI * nfib / goldenRatio; + cost = 1.0 - 2.0 * nfib/Nfib; + cost = (cost < -1.0 ? -1.0 : (cost > 1.0 ? 1.0 : cost)); + invt = return_euler(phi, acos(cost), 0, 1); + } + + // Update rotation matrix + rot(0, 0) = rot(1, 1) = cos(Q); + rot(0, 1) = -sin(Q); + rot(1, 0) = sin(Q); + + // Full rotation matrix + Eigen::Matrix3d trans; + if (Nfib>1) trans = invt * rot * proj; + else trans = Iprj * rot * proj; + + // Rotate position and velocity + pos1 = trans * pos; + vel1 = -trans * vel; + + // Reset the phase-space vector + // + ps[0] = mfac; + + ps[1] = pos1(0); + ps[2] = pos1(1); + ps[3] = pos1(2); + + ps[4] = vel1(0); + ps[5] = vel1(1); + ps[6] = vel1(2); + } } - if (myid==0 and !((n+1)%NREPORT)) cout << '\r' << (n+1)*numprocs << flush; + if (myid==0 and !((n+1)%NREPORT)) cout << '\r' << (n+1)*numprocs*Nrepl + << flush; } if (zeropos or zerovel) { @@ -864,16 +1019,20 @@ main(int argc, char **argv) MPI_Reduce(MPI_IN_PLACE, &WW, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); MPI_Reduce(MPI_IN_PLACE, &VC, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); - cout << std::endl << "-----------" << std::endl << std::endl; - cout << std::setw(20) << "States rejected: " << count << std::endl; - cout << std::setw(20) << "Negative masses: " << negms << std::endl; - cout << std::setw(60) << setfill('-') << '-' << std::endl << setfill(' '); - cout << std::setw(20) << "KE=" << TT << std::endl; - cout << std::setw(20) << "PE=" << WW << std::endl; - cout << std::setw(20) << "VC=" << VC << std::endl; - cout << std::setw(20) << "Ratio (-2T/W)=" << -2.0*TT/WW << std::endl; - cout << std::setw(20) << "Ratio (-2T/C)=" << -2.0*TT/VC << std::endl; - std::cout << std::setw(60) << std::setfill('-') << '-' << std::endl << std::setfill(' '); + std::cout << std::endl + << std::setw(60) << std::setfill('-') << '-' << std::endl + << std::setfill(' ') + << std::setw(20) << "States rejected: " << count << std::endl + << std::setw(20) << "Negative masses: " << negms << std::endl + << std::setw(60) << setfill('-') << '-' << std::endl + << setfill(' ') + << std::setw(20) << "KE=" << TT << std::endl + << std::setw(20) << "PE=" << WW << std::endl + << std::setw(20) << "VC=" << VC << std::endl + << std::setw(20) << "Ratio (-2T/W)=" << -2.0*TT/WW << std::endl + << std::setw(20) << "Ratio (-2T/C)=" << -2.0*TT/VC << std::endl + << std::setw(60) << std::setfill('-') << '-' << std::endl + << std::setfill(' '); } out.close(); From 8539dc51ceceb3144e1ff2020e1e55ad3405d641 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 21 Jan 2025 15:28:57 -0500 Subject: [PATCH 007/106] Make goldenRatio a const rather than a constexpr --- exputil/realize_model.cc | 3 ++- utils/ICs/gensph.cc | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/exputil/realize_model.cc b/exputil/realize_model.cc index 3b2361adb..14ffc230c 100644 --- a/exputil/realize_model.cc +++ b/exputil/realize_model.cc @@ -799,7 +799,8 @@ AxiSymModel::PSret AxiSymModel::gen_point_3d_iso // Fibonacci lattice for velocities // - constexpr double goldenRatio = 0.5*(1.0 + sqrt(5.0)); + const double goldenRatio = 1.618033988749895; + double vphi = 2.0*M_PI * ja / goldenRatio; double vcost = 1.0 - 2.0*ja/na; double vsint = sqrt(fabs(1.0 - vcost*vcost)); diff --git a/utils/ICs/gensph.cc b/utils/ICs/gensph.cc index 9541ee21f..c5aa441a7 100644 --- a/utils/ICs/gensph.cc +++ b/utils/ICs/gensph.cc @@ -78,6 +78,8 @@ main(int argc, char **argv) bool VTEST; std::string INFILE, MMFILE, OUTFILE, OUTPS, config; + const double goldenRatio = 1.618033988749895; + #ifdef DEBUG set_fpu_handler(); #endif @@ -770,8 +772,6 @@ main(int argc, char **argv) } } - constexpr double goldenRatio = 0.5*(1.0 + sqrt(5.0)); - for (int n=beg; n Date: Fri, 24 Jan 2025 13:18:32 -0500 Subject: [PATCH 008/106] Print mismatched cache variables --- exputil/EmpCyl2d.cc | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/exputil/EmpCyl2d.cc b/exputil/EmpCyl2d.cc index efd29bcdf..fac8f21b5 100644 --- a/exputil/EmpCyl2d.cc +++ b/exputil/EmpCyl2d.cc @@ -780,19 +780,31 @@ bool EmpCyl2d::ReadH5Cache() auto checkInt = [&file](int value, std::string name) { int v; HighFive::Attribute vv = file.getAttribute(name); vv.read(v); - if (value == v) return true; return false; + if (value == v) return true; + if (myid==0) std::cout << "---- EmpCyl2d::ReadH5Cache: " + << name << " expected " << value << " but found " + << v << std::endl; + return false; }; auto checkDbl = [&file](double value, std::string name) { double v; HighFive::Attribute vv = file.getAttribute(name); vv.read(v); - if (fabs(value - v) < 1.0e-16) return true; return false; + if (fabs(value - v) < 1.0e-16) return true; + if (myid==0) std::cout << "---- EmpCyl2d::ReadH5Cache: " + << name << " expected " << value << " but found " + << v << std::endl; + return false; }; auto checkStr = [&file](std::string value, std::string name) { std::string v; HighFive::Attribute vv = file.getAttribute(name); vv.read(v); - if (value.compare(v)==0) return true; return false; + if (value.compare(v)==0) return true; + if (myid==0) std::cout << "---- EmpCyl2d::ReadH5Cache: " + << name << " expected " << value << " but found " + << v << std::endl; + return false; }; // From 95d9a9985542d32501a7c55ac8cca31befcabd23 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Fri, 24 Jan 2025 13:18:56 -0500 Subject: [PATCH 009/106] Add quiet-start method --- utils/ICs/ZangICs.cc | 52 +++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/utils/ICs/ZangICs.cc b/utils/ICs/ZangICs.cc index 638c6c755..660a7633c 100644 --- a/utils/ICs/ZangICs.cc +++ b/utils/ICs/ZangICs.cc @@ -27,6 +27,7 @@ main(int ac, char **av) //===================== int N; // Number of particles + int Nrepl; // Number of particle replicates per orbit double mu, nu, Ri, Ro; // Taper paramters double Rmin, Rmax; // Radial range double sigma; // Velocity dispersion @@ -59,6 +60,8 @@ main(int ac, char **av) cxxopts::value(sigma)->default_value("1.0")) ("s,seed", "Random number seed. Default: use /dev/random", cxxopts::value(seed)) + ("q,Nrepl", "Number of particle replicates per orbit", + cxxopts::value(Nrepl)->default_value("1")) ("f,file", "Output body file", cxxopts::value(bodyfile)->default_value("zang.bods")) ; @@ -84,6 +87,10 @@ main(int ac, char **av) seed = std::random_device{}(); } + // Set particle number consitent with even replicants + if (Nrepl<1) Nrepl = 1; + if (Nrepl>1) N = (N/Nrepl)*Nrepl; + // Make sure N>0 if (N<=0) { std::cerr << av[0] << ": you must requiest at least one body" @@ -107,6 +114,11 @@ main(int ac, char **av) 1.0, Rmin, Rmax); model->setup_df(sigma); + // Replicant logic + // + int Number = N/Nrepl; + double dPhi = 2.0*M_PI/Nrepl; + // Progress bar // std::shared_ptr progress; @@ -115,7 +127,7 @@ main(int ac, char **av) { nomp = omp_get_num_threads(); if (omp_get_thread_num()==0) { - progress = std::make_shared(N/nomp); + progress = std::make_shared(Number/nomp); } } @@ -190,7 +202,7 @@ main(int ac, char **av) // Generation loop with OpenMP // #pragma omp parallel for reduction(+:over) - for (int n=0; n M_PI) vr *= -1.0; // Branch of radial motion - // Convert from polar to Cartesian - // - pos[n][0] = r*cos(phi); - pos[n][1] = r*sin(phi); - pos[n][2] = 0.0; - - vel[n][0] = vr*cos(phi) - vt*sin(phi); - vel[n][1] = vr*sin(phi) + vt*cos(phi); - vel[n][2] = 0.0; - - // Accumulate mean position and velocity - // - for (int k=0; k<3; k++) { - zeropos[tid][k] += pos[n][k]; - zerovel[tid][k] += vel[n][k]; + for (int nn=0; nn Date: Tue, 4 Feb 2025 16:29:23 -0500 Subject: [PATCH 010/106] Report cache name tagged for recomputation --- exputil/EmpCyl2d.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exputil/EmpCyl2d.cc b/exputil/EmpCyl2d.cc index fac8f21b5..37f36cfb0 100644 --- a/exputil/EmpCyl2d.cc +++ b/exputil/EmpCyl2d.cc @@ -815,8 +815,8 @@ bool EmpCyl2d::ReadH5Cache() if (not checkStr(Version, "Version")) return false; } else { if (myid==0) - std::cout << "---- EmpCyl2d::ReadH5Cache: " - << "recomputing cache for HighFive API change" + std::cout << "---- EmpCyl2d::ReadH5Cache: " << "<" << cache_name_2d + << "> recomputing cache for HighFive API change" << std::endl; return false; } From 72fc23b17fa5735a827f55dbb10d5271458b8a0a Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 19 Feb 2025 19:12:28 -0500 Subject: [PATCH 011/106] Removed the angle-only quick start for the sphere, which leaves too much even-order noise to be worth keeping --- utils/ICs/gensph.cc | 78 +++------------------------------------------ 1 file changed, 4 insertions(+), 74 deletions(-) diff --git a/utils/ICs/gensph.cc b/utils/ICs/gensph.cc index c5aa441a7..045d48f28 100644 --- a/utils/ICs/gensph.cc +++ b/utils/ICs/gensph.cc @@ -69,7 +69,7 @@ int main(int argc, char **argv) { int HMODEL, N, NUMDF, NUMR, NUMJ, NUME, NUMG, NREPORT, SEED, ITMAX; - int NUMMODEL, RNUM, DIVERGE, DIVERGE2, LINEAR, NUMINT, NI, ND, Nangle; + int NUMMODEL, RNUM, DIVERGE, DIVERGE2, LINEAR, NUMINT, NI, ND; int Nrepl=1, Nfib=1; double DIVERGE_RFAC, DIVERGE_RFAC2, NN, MM, RA, RMODMIN, RMOD, EPS; double X0, Y0, Z0, U0, V0, W0, TOLE; @@ -184,8 +184,6 @@ main(int argc, char **argv) cxxopts::value(ELIMIT)->default_value("false")) ("VTEST", "Test gen_velocity() generation", cxxopts::value(VTEST)->default_value("false")) - ("Nangle", "Number of angular points on a regular spherical grid (0 skips this algorithm). Nangle>0 takes precidence over other algorithms.", - cxxopts::value(Nangle)->default_value("0")) ("Nrepl", "Number of replicates in orbital plane (1 skips the Sellwood 1997 algorithm)", cxxopts::value(Nrepl)->default_value("1")) ("Nfib", "Number of points on the sphere for each orbit. Replicate the orbits by tiling the angular momentum direction on the sphere using a Fibonnaci sequence. Default value of 1 implies one plane per orbit.", @@ -267,13 +265,8 @@ main(int argc, char **argv) // Use Fibonnaci for space-filling grid points // - if (Nangle>0) { - Nfib = Nrepl = 1; - } - else { - Nfib = std::max(1, Nfib ); - Nrepl = std::max(1, Nrepl); - } + Nfib = std::max(1, Nfib ); + Nrepl = std::max(1, Nrepl); // Prepare output streams and create new files // @@ -542,20 +535,6 @@ main(int argc, char **argv) } int nfib=0, nangle=0, nshell=0; - if (Nangle) { - nangle = Nangle; - nfib = floor(sqrt(double(N)/(nangle*nangle))); - nshell = nfib*nangle; - - // Shells in radius and velocity - // - N = nshell*nshell; - if (myid==0) - std::cout << std::setw(60) << std::setfill('-') << '-' - << std::setfill(' ') << std::endl - << "Fibonacci: N=" << N << " Nangle=" << Nangle - << " Nmag=" << nfib << " Nshell=" << nshell<< std::endl; - } // For replication algorithm; Nrepl=1 is the standard algorithm. // @@ -756,60 +735,11 @@ main(int argc, char **argv) Eigen::VectorXd zz = Eigen::VectorXd::Zero(7); std::vector rmass; - if (Nangle) { - rmass.resize(nfib); - - // Set up mass grid - double rmin = hmodel->get_min_radius(); - double rmax = hmodel->get_max_radius(); - double mmas = hmodel->get_mass(rmax); - double dmas = mmas/nfib; - for (int i=0; i double - { return (mas - hmodel->get_mass(r)); }; - rmass[i] = zbrent(loc, rmin, rmax, 1.0e-8); - } - } for (int n=beg; n=0 && nspace=0 && nveloc=0 && rmgen_point_iso(r, acos(cost), phi, - nveloc, nfib, nangle, - PHI, THETA, PHI); - - for (int i=0; i<3; i++) { - if (std::isnan(ps[i+3])) { - std::cout << "Vel NaN with r=" << r << " cost=" << cost << " phi=" << phi << std::endl; - } - } - } - else if (ELIMIT) + if (ELIMIT) std::tie(ps, ierr) = rmodel->gen_point(Emin0, Emax0, Kmin0, Kmax0); else if (VTEST) { std::tie(ps, ierr) = rmodel->gen_point(); From 4ab47e9c5c0fadbd26a3264de719d2f9118163c1 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 15 Mar 2025 11:27:58 -0400 Subject: [PATCH 012/106] Added logic to cache and fix the monopole coefficients --- src/SphericalBasis.H | 6 ++++++ src/SphericalBasis.cc | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/SphericalBasis.H b/src/SphericalBasis.H index cd949a96e..10b3a7899 100644 --- a/src/SphericalBasis.H +++ b/src/SphericalBasis.H @@ -232,6 +232,12 @@ protected: //! Flag self_consitency bool self_consistent; + //! Freeze monopole + bool FIX_L0; + + //! Frozen monopole coefficients + Eigen::VectorXd C0; + //! Flag fixed monopole bool NO_L0; diff --git a/src/SphericalBasis.cc b/src/SphericalBasis.cc index 6a0480147..7470b92b8 100644 --- a/src/SphericalBasis.cc +++ b/src/SphericalBasis.cc @@ -31,6 +31,7 @@ SphericalBasis::valid_keys = { "rmin", "rmax", "self_consistent", + "FIX_L0", "NO_L0", "NO_L1", "EVEN_L", @@ -66,6 +67,7 @@ SphericalBasis::SphericalBasis(Component* c0, const YAML::Node& conf, MixtureBas mix = m; geometry = sphere; coef_dump = true; + FIX_L0 = false; NO_L0 = false; NO_L1 = false; EVEN_L = false; @@ -111,6 +113,7 @@ SphericalBasis::SphericalBasis(Component* c0, const YAML::Node& conf, MixtureBas } else self_consistent = true; + if (conf["FIX_L0"]) FIX_L0 = conf["FIX_L0"].as(); if (conf["NO_L0"]) NO_L0 = conf["NO_L0"].as(); if (conf["NO_L1"]) NO_L1 = conf["NO_L1"].as(); if (conf["EVEN_L"]) EVEN_L = conf["EVEN_L"].as(); @@ -1646,6 +1649,14 @@ void SphericalBasis::determine_acceleration_and_potential(void) } + // This should suffice for both CPU and GPU + if (FIX_L0) { + // Save the monopole coefficients on the first evaluation + if (C0.size() == 0) *C0 = expcoef[0]; + // Copy the saved coefficients to the active array + else *expcoef[0] = C0; + } + #if HAVE_LIBCUDA==1 if (use_cuda and cC->cudaDevice>=0 and cC->force->cudaAware()) { if (cudaAccelOverride) { From 9a0a5602319321e322dd791212aa9dd30891bffe Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 15 Mar 2025 11:45:23 -0400 Subject: [PATCH 013/106] Typo fix --- src/SphericalBasis.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SphericalBasis.cc b/src/SphericalBasis.cc index 7470b92b8..4cde80982 100644 --- a/src/SphericalBasis.cc +++ b/src/SphericalBasis.cc @@ -1652,7 +1652,7 @@ void SphericalBasis::determine_acceleration_and_potential(void) // This should suffice for both CPU and GPU if (FIX_L0) { // Save the monopole coefficients on the first evaluation - if (C0.size() == 0) *C0 = expcoef[0]; + if (C0.size() == 0) C0 = *expcoef[0]; // Copy the saved coefficients to the active array else *expcoef[0] = C0; } From cd8cbde2bd598af8cd8b850c4513fb5bce92b440 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 25 Mar 2025 15:33:10 -0400 Subject: [PATCH 014/106] Add KDE 2d smoothing to diffpsp --- utils/PhaseSpace/CMakeLists.txt | 2 +- utils/PhaseSpace/KDE2d.H | 95 ++++++++++++ utils/PhaseSpace/KDE2d.cc | 262 ++++++++++++++++++++++++++++++++ utils/PhaseSpace/diffpsp.cc | 41 ++++- 4 files changed, 396 insertions(+), 4 deletions(-) create mode 100644 utils/PhaseSpace/KDE2d.H create mode 100644 utils/PhaseSpace/KDE2d.cc diff --git a/utils/PhaseSpace/CMakeLists.txt b/utils/PhaseSpace/CMakeLists.txt index 63ced2c1f..809835df7 100644 --- a/utils/PhaseSpace/CMakeLists.txt +++ b/utils/PhaseSpace/CMakeLists.txt @@ -66,7 +66,7 @@ add_executable(psp2rings psp2rings.cc PSP.cc) add_executable(psp2bess psp2bess.cc PSP.cc Bess.cc) add_executable(psp2lagu psp2lagu.cc PSP.cc) add_executable(psp2interp psp2interp.cc PSP.cc) -add_executable(diffpsp diffpsp.cc MakeModel.cc PSP.cc) +add_executable(diffpsp diffpsp.cc MakeModel.cc PSP.cc KDE2d.cc) add_executable(pspmono pspmono.cc MakeModel.cc PSP.cc) add_executable(psp2hdf5 psp2hdf5.cc PSP.cc) add_executable(psporbv psporbv.cc PSP.cc) diff --git a/utils/PhaseSpace/KDE2d.H b/utils/PhaseSpace/KDE2d.H new file mode 100644 index 000000000..9b3967224 --- /dev/null +++ b/utils/PhaseSpace/KDE2d.H @@ -0,0 +1,95 @@ +#ifndef KDE2D_H +#define KDE2D_H + +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include + +namespace KDE +{ + class KDE2d + { + protected: + + //@{ + //! Parameters + bool debug = false; + const int minKsize = 7; + int numx, numy; + double xmin, xmax, ymin, ymax, sigmax, sigmay, delx, dely; + //@} + + //! Create a smooth 2D gaussian kernel + std::vector + gaussian_kernel_2d(int xsize, int ysize); + + //! Do the FFT convolution + void kde_fft_2d(); + + //@{ + //! Make 2D grid + void grid_pairs(const std::vector>& data); + void grid_array(const std::vector& x, const std::vector& y); + //@} + + //! Eigen matrices + Eigen::MatrixXd grid, smooth; + + public: + + //! Constructor + KDE2d(int numx, int numy, + double xmin, double xmax, + double ymin, double ymax, + double sigmax, double sigmay) : + numx(numx), numy(numy), xmin(xmin), xmax(xmax), + ymin(ymin), ymax(ymax), sigmax(sigmax), sigmay(sigmay) + { + delx = xmax - xmin; + dely = ymax - ymin; + if (delx<=0 || dely<=0) + throw std::invalid_argument("Grid size must be positive"); + } + + //! Get the smoothed density from an input grid + const Eigen::MatrixXd& operator()(const Eigen::MatrixXd& GRID) + { + if (GRID.rows() != numx || GRID.cols() != numy) + throw std::invalid_argument("Grid size must match KDE2d object"); + grid = GRID; + kde_fft_2d(); + return smooth; + } + + //! Get the smoothed density from an input data pairs + const Eigen::MatrixXd& operator()(const std::vector>& data) + { + grid_pairs(data); + kde_fft_2d(); + return smooth; + } + + //! Get the smoothed density from an input data arrays + const Eigen::MatrixXd& operator() + (const std::vector& x, const std::vector& y) + { + grid_array(x, y); + kde_fft_2d(); + return smooth; + } + + //! Set debug mode + void setDebug() { debug = true; } + }; +} + +#endif // KDE2D_H diff --git a/utils/PhaseSpace/KDE2d.cc b/utils/PhaseSpace/KDE2d.cc new file mode 100644 index 000000000..1218d4b1c --- /dev/null +++ b/utils/PhaseSpace/KDE2d.cc @@ -0,0 +1,262 @@ +#include + +namespace KDE +{ + // Function to compute the Gaussian kernel in pixel units + // + std::vector KDE2d::gaussian_kernel_2d(int xsize, int ysize) + { + std::vector kernel(xsize * ysize); + double sum = 0.0; + for (int i = -xsize / 2; i < xsize / 2 + 1; i++) { + + double dx = delx*i/numx; + double facx = exp(-0.5*dx*dx/(sigmax*sigmax)); + + for (int j = -ysize / 2; j < ysize / 2 + 1; j++) { + + double dy = dely*j/numy; + double facy = exp(-0.5*dy*dy/(sigmay*sigmay)); + + kernel[(i + xsize / 2) * ysize + (j + ysize / 2)] = facx * facy; + + sum += facx * facy; + } + } + + // Normalize the kernel + // + for (auto & v : kernel) v /= sum; + + return kernel; + } + + + void KDE2d::kde_fft_2d() + { + // Sanity check + // + if (numx != grid.rows() || numy != grid.cols()) + throw std::runtime_error("Invalid grid dimensions"); + + // Compute the Gaussian kernel + // + int kern_xsize = std::max(minKsize, + std::floor(sigmax/delx*numx*minKsize)); + + if (kern_xsize % 2 == 0) kern_xsize += 1; // Make the size odd + + int kern_ysize = std::max(minKsize, + std::floor(sigmay/dely*numy*minKsize)); + + if (kern_ysize % 2 == 0) kern_ysize += 1; // Make the size odd + + std::vector kern = gaussian_kernel_2d(kern_xsize, kern_ysize); + + if (debug) { + std::ofstream fout("kern0.dat"); + fout << kern_xsize << " " << kern_ysize << std::endl; + for (int j=0; j padded_grid(padded_xsize * padded_ysize, 0.0); + std::vector padded_kern(padded_xsize * padded_ysize, 0.0); + + for (int i=0; i>& data) + { + // Sanity check + // + if (data.size() == 0) + throw std::invalid_argument("data must be non-empty"); + + // Create a grid to evaluate KDE + // + grid.resize(numx, numy); + grid.setZero(); + for (const auto& point : data) { + int x = static_cast( (point.first - xmin)/delx * numx); + int y = static_cast( (point.second - ymin)/dely * numy); + + if (x >= 0 && x < numx && y >= 0 && y < numy) { + grid(x, y) += 1.0; + } + else { + std::cout << "Out of range: " << point.first << " " << point.second << std::endl; + } + } + } + + void + KDE2d::grid_array(const std::vector& X, const std::vector& Y) + { + // Sanity check + // + if (X.size() != Y.size()) + throw std::invalid_argument("x and y must be the same size"); + + if (X.size() == 0) + throw std::invalid_argument("data must be non-empty"); + + // Create a grid to evaluate KDE + // + grid.resize(numx, numy); + grid.setZero(); + for (int i=0; i= 0 && x < numx && y >= 0 && y < numy) { + grid(x, y) += 1.0; + } + else { + std::cout << "Out of range: " << X[i] << " " << Y[i] << std::endl; + } + } + } + +} +// END namespace KDE diff --git a/utils/PhaseSpace/diffpsp.cc b/utils/PhaseSpace/diffpsp.cc index 42890f9c6..e396d9997 100755 --- a/utils/PhaseSpace/diffpsp.cc +++ b/utils/PhaseSpace/diffpsp.cc @@ -54,6 +54,7 @@ #include // Enhanced option parsing #include // EXP library globals #include +#include "KDE2d.H" // Kernel density estimation void p_rec(std::ofstream& out, double E, double K, double V) { @@ -96,6 +97,7 @@ main(int argc, char **argv) double EMAX; double KMIN; double KMAX; + double Sig1, Sig2; double I1min=-1, I1max=-1, I2min=-1, I2max=-1; int NLIMIT; int NSKIP; @@ -241,6 +243,10 @@ main(int argc, char **argv) cxxopts::value(DIVERGE_RFAC)->default_value("1.0")) ("Kpow", "Create kappa bins with power scaling", cxxopts::value(KPOWER)->default_value("1.0")) + ("Sig1", "Bin smoothing in Dimension 1 (must be > 0)", + cxxopts::value(Sig1)->default_value("0.0")) + ("Sig2", "Bin smoothing in Dimension 2 (must be > 0)", + cxxopts::value(Sig2)->default_value("0.0")) ("INFILE1", "Fiducial phase-space file", cxxopts::value>(INFILE1)) ("INFILE2", "Evolved phase-space file", @@ -513,7 +519,7 @@ main(int argc, char **argv) if (myid==0) std::cout << std::endl << std::string(40, '-') << std::endl - << "---- Action limits" << std::endl + << "---- Range limits" << std::endl << std::string(40, '-') << std::endl << "-- I1min: " << I1min << std::endl << "-- I1max: " << I1max << std::endl @@ -521,10 +527,21 @@ main(int argc, char **argv) << "-- I2max: " << I2max << std::endl << std::string(40, '-') << std::endl; } else { - if (I1min<0) I1min = Emin; - if (I1max<0) I1max = Emax; + if (I1min>0) I1min = Emin; + if (I1max>0) I1max = Emax; if (I2min<0) I2min = std::max(KTOL, KMIN); if (I2max<0) I2max = std::min(1.0 - KTOL, KMAX); + + if (myid==0) + std::cout << std::endl + << std::string(40, '-') << std::endl + << "---- Range limits" << std::endl + << std::string(40, '-') << std::endl + << "-- Emin: " << I1min << std::endl + << "-- Emax: " << I1max << std::endl + << "-- Kmin: " << I2min << std::endl + << "-- Kmax: " << I2max << std::endl + << std::string(40, '-') << std::endl; } double d1 = (I1max - I1min) / NUM1; @@ -1249,6 +1266,24 @@ main(int argc, char **argv) bool relJ = false; if (vm.count("relJ")) relJ = true; + // Smooth arrays + // + if (Sig1 > 0.0 and Sig2 > 0.0) { + + KDE::KDE2d kde(NUM1, NUM2, + I1min, I1max, I2min, I2max, Sig1, Sig2); + + histoM = kde(histoM); + histoC = kde(histoC); + histoE = kde(histoE); + histoJ = kde(histoJ); + histoR = kde(histoR); + histoI = kde(histoI); + histoT = kde(histoT); + histoF = kde(histoF); + histoDF = kde(histoDF); + } + for (int j=0; j Date: Tue, 25 Mar 2025 18:56:02 -0400 Subject: [PATCH 015/106] Comment improvements only --- utils/PhaseSpace/KDE2d.H | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/utils/PhaseSpace/KDE2d.H b/utils/PhaseSpace/KDE2d.H index 9b3967224..d8fa41f27 100644 --- a/utils/PhaseSpace/KDE2d.H +++ b/utils/PhaseSpace/KDE2d.H @@ -16,14 +16,38 @@ namespace KDE { + /** + Class for 2D kernel density estimation + + Parameters + ---------- + @param numx: desired grid size in x + @param numy: desired grid size in y + @param xmin: minimum value in x + @param xmax: maximum value in x + @param ymin: minimum value in y + @param ymax: maximum value in y + @param sigmax: smoothing kernel width in x units + @param sigmay: smoothing kernel width in y units + + Data input + ---------- + Can be either a vector of pairs (x,y), an array of x and y + values, or a precomputed histogram-like grid of size (numx, + numy). + + Output + ------ + Smoothed 2D grid in Eigen::MatrixXd format + */ class KDE2d { protected: //@{ //! Parameters - bool debug = false; - const int minKsize = 7; + bool debug = false; // Set to true for intermediate output + const int minKsize = 7; // Default minimum kernel size int numx, numy; double xmin, xmax, ymin, ymax, sigmax, sigmay, delx, dely; //@} From ae7521100d50917a4112d01f597f84237fd6b0a7 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 1 Apr 2025 15:53:01 -0400 Subject: [PATCH 016/106] Preliminary Cuda implementation for UserBar --- src/user/CMakeLists.txt | 1 + src/user/UserBar.H | 5 ++ src/user/UserBar.cc | 18 ++-- src/user/cudaUserBar.cu | 192 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 6 deletions(-) create mode 100644 src/user/cudaUserBar.cu diff --git a/src/user/CMakeLists.txt b/src/user/CMakeLists.txt index 1f7b4c85a..d199067d1 100644 --- a/src/user/CMakeLists.txt +++ b/src/user/CMakeLists.txt @@ -34,6 +34,7 @@ set(mwgala_SRC UserMW.cc) if(ENABLE_CUDA) set(cudatest_SRC UserTestCuda.cc cudaUserTest.cu) + list(APPEND barSRC cudauserBar.cu) endif() foreach(mlib ${USER_MODULES}) diff --git a/src/user/UserBar.H b/src/user/UserBar.H index 81136cd7f..2e4a81cd7 100644 --- a/src/user/UserBar.H +++ b/src/user/UserBar.H @@ -31,6 +31,11 @@ private: void * determine_acceleration_and_potential_thread(void * arg); void initialize(); +#if HAVE_LIBCUDA==1 + //! Cuda implementation + void determine_acceleration_and_potential_cuda(); +#endif + double length, bratio, cratio, amplitude, Ton, Toff, DeltaT, Fcorot; bool fixed, soft; string filename; diff --git a/src/user/UserBar.cc b/src/user/UserBar.cc index db3f6eeed..458fe6e4b 100644 --- a/src/user/UserBar.cc +++ b/src/user/UserBar.cc @@ -30,7 +30,9 @@ UserBar::UserBar(const YAML::Node &conf) : ExternalForce(conf) ctr_name = ""; // Default component for com angm_name = ""; // Default component for angular momentum - initialize(); + initialize(); // Assign parameters + + cuda_aware = true; // Cuda routine is implemented if (ctr_name.size()>0) { // Look for the fiducial component @@ -175,10 +177,6 @@ void UserBar::determine_acceleration_and_potential(void) // Write to bar state file, if true bool update = false; -#if HAVE_LIBCUDA==1 // Cuda compatibility - getParticlesCuda(cC); -#endif - if (c1) { c1->get_angmom(); // Tell component to compute angular momentum // cout << "Lz=" << c1->angmom[2] << endl; // debug @@ -289,7 +287,6 @@ void UserBar::determine_acceleration_and_potential(void) << endl; } } - Iz = 0.2*mass*(a1*a1 + a2*a2); Lz = Iz * omega; @@ -390,7 +387,16 @@ void UserBar::determine_acceleration_and_potential(void) } } +#if HAVE_LIBCUDA==1 // Cuda compatibility + if (use_cuda) { + determine_acceleration_and_potential_cuda(); + } else { + getParticlesCuda(cC); // Cuda override + exp_thread_fork(false); + } +#else exp_thread_fork(false); +#endif if (myid==0 && update) { diff --git a/src/user/cudaUserBar.cu b/src/user/cudaUserBar.cu new file mode 100644 index 000000000..bde7ce726 --- /dev/null +++ b/src/user/cudaUserBar.cu @@ -0,0 +1,192 @@ +// -*- C++ -*- + +#include +#include + +#include +#include "UserSat.H" + +// Global device symbols for CUDA kernel +// +__device__ __constant__ +cuFP_t userBarAmp, userBarPosAng, userBarCen[3], userBarB5; + +__device__ __constant__ +int userBarSoft; + + +// Cuda implementation of bar force +// +__global__ void +userBarForceKernel(dArray P, dArray I, + int stride, PII lohi) +{ + const int tid = blockDim.x * blockIdx.x + threadIdx.x; + + for (int n=0; n=P._s) printf("out of bounds: %s:%d\n", __FILE__, __LINE__); +#endif + cudaParticle & p = P._v[I._v[npart]]; + + // Compute position relative to bar + cuFP_t rr = 0.0, fac, ffac, nn; + for (int k=0; k<3; k++) { + fpos[k] = p.pos[k] - userBarCen[k]; + rr += fpos[k] * fpos[k]; + } + rr = sqrt(rr); + + cuFP_t xx = fpos[0]; + cuFP_t yy = fpos[1]; + cuFP_t zz = fpos[2]; + cuFP_t pp = (xx*xx - yy*yy)*cos2p + 2.0*xx*yy*sin2p; + + if (userBarSoft) { + fac = 1.0 + rr/b5; + ffac = -userBarAmp / pow(fac, 6.0); + nn = pp / (b5*rr); + } else { + fac = 1.0 + pow(rr/b5, 5.0); + ffac = -userBarAmp / (fac*fac); + nn = pp * pow(rr/b5, 3.0)/ (b5*b5); + } + + // Add acceleration + p.acc[0] += ffac * ( 2.0*( xx*cos2p + yy*sin2p)*fac - 5.0*nn*xx); + p.acc[1] += ffac * ( 2.0*(-yy*cos2p + xx*sin2p)*fac - 5.0*nn*yy); + p.acc[2] += ffac * ( -5.0*nn*zz); + + // Add external potential + p.potext += -ffac*pp*fac; + + } // Particle index block + + } // END: stride loop + +} + + +__global__ +void testConstantsUserSat(cuFP_t tnow) +{ + printf("-------------------------\n"); + printf("---UserSat constants-----\n"); + printf("-------------------------\n"); + printf(" Time = %e\n", tnow ); + printf(" Amp = %e\n", userBarAmp ); + printf(" Amp = %e\n", userBarNumFac ); + printf(" b5 = %e\n", userBarB5 ); + printf(" Center = %e, %e, %e\n", + userBarCen[0], userBarCen[1], userBarCen[2] ); + if (userBarSoft) printf(" Soft = true\n" ); + else printf(" Soft = false\n"); + printf("-------------------------\n"); +} + +void UserBar::determine_acceration_and_potential_cuda() +{ + // Sanity check + // + int nbodies = cC->Number(); + if (nbodies != static_cast(cC->Particles().size())) { + std::cerr << "UserSat: ooops! number=" << nbodies + << " but particle size=" << cC->Particles().size() << endl; + nbodies = static_cast(cC->Particles().size()); + } + + if (nbodies==0) { // Return if there are no particles + if (verbose) { + cout << "Process " << myid << ": in UserBar, nbodies=0" + << " for Component <" << cC->name << "> at T=" << tnow + << endl; + } + return; + } + + double amp = afac * numfac * amplitude/fabs(amplitude) + * 0.5*(1.0 + erf( (tnow - Ton )/DeltaT )) + * 0.5*(1.0 - erf( (tnow - Toff)/DeltaT )) ; + + cudaDeviceProp deviceProp; + cudaGetDeviceProperties(&deviceProp, cC->cudaDevice); + cuda_check_last_error_mpi("cudaGetDeviceProperties", __FILE__, __LINE__, myid); + + // Stream structure iterators + // + auto cr = cC->cuStream; + + // Assign expansion center + // + auto cn = cC->getCenter(); + cuFP_t ctr[3], dtmp; + for (int k=0; k<3; k++) ctr[k] = cn[k]; + + cuFP_t dtmp; + + cuda_safe_call(cudaMemcpyToSymbol(userBarAmp, &(dtmp=amp), sizeof(cuFP_t), + size_t(0), cudaMemcpyHostToDevice), + __FILE__, __LINE__, "Error copying userBarAmp"); + + cuda_safe_call(cudaMemcpyToSymbol(userBarPosAng, &(dtmp=posang), sizeof(cuFP_t), + size_t(0), cudaMemcpyHostToDevice), + __FILE__, __LINE__, "Error copying userBarPosAng"); + + cuda_safe_call(cudaMemcpyToSymbol(userBarCen, ctr, sizeof(cuFP_t)*3, + size_t(0), cudaMemcpyHostToDevice), + __FILE__, __LINE__, "Error copying userBarCen"); + + cuda_safe_call(cudaMemcpyToSymbol(userBarB5, &(dtmp=b5), sizeof(cuFP_t), + size_t(0), cudaMemcpyHostToDevice), + __FILE__, __LINE__, "Error copying userBarB5"); + + int cuSoft = 0; + if (soft) cuSoft = 1; + cuda_safe_call(cudaMemcpyToSymbol(userBarSoft, &cuSoft, sizeof(int), + size_t(0), cudaMemcpyHostToDevice), + __FILE__, __LINE__, "Error copying userBarSoft"); + + // VERBOSE diagnostic output on first ncheck calls + // + static int cntr = 0; + const int ncheck = 100; + + if (myid==0 and VERBOSE>4 and cntr < ncheck) { + testConstantsUserBar<<<1, 1, 0, cr->stream>>>(tnow); + cudaDeviceSynchronize(); + cuda_check_last_error_mpi("cudaDeviceSynchronize", __FILE__, __LINE__, myid); + cntr++; + } + + // Get particle index range for levels [mlevel, multistep] + // + PII lohi = cC->CudaGetLevelRange(mlevel, multistep); + + // Compute grid + // + unsigned int N = lohi.second - lohi.first; + unsigned int stride = N/BLOCK_SIZE/deviceProp.maxGridSize[0] + 1; + unsigned int gridSize = N/BLOCK_SIZE/stride; + + if (N>0) { + + if (N > gridSize*BLOCK_SIZE*stride) gridSize++; + + // Do the work + // + userBarForceKernel<<stream>>> + (toKernel(cr->cuda_particles), toKernel(cr->indx1), stride, lohi); + } +} From ab77174cb0e7f6d467f6b3783dd082f21b20f8b6 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 1 Apr 2025 16:35:00 -0400 Subject: [PATCH 017/106] Simply CUDA stanza in CMakeLists.txt --- src/user/CMakeLists.txt | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/user/CMakeLists.txt b/src/user/CMakeLists.txt index d199067d1..e89148314 100644 --- a/src/user/CMakeLists.txt +++ b/src/user/CMakeLists.txt @@ -12,6 +12,14 @@ set (common_INCLUDE_DIRS set (common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX exputil EXPlib yaml-cpp) +set(user_SRC UserTest.cc) +set(bar_SRC UserBar.cc) +set(logpot_SRC UserLogPot.cc) +set(disk_SRC UserDisk.cc) +set(halo_SRC UserHalo.cc) +set(mndisk_SRC UserMNdisk.cc) +set(mwgala_SRC UserMW.cc) + if(ENABLE_CUDA) list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) @@ -22,19 +30,8 @@ if(ENABLE_CUDA) # set_source_files_properties(UserBar.cc UserDisk.cc UserHalo.cc # UserLogPot.cc UserMNdisk.cc UserMW.cc UserTest.cc UserTestCuda.cc # PROPERTIES LANGUAGE CUDA) -endif() - -set(user_SRC UserTest.cc) -set(bar_SRC UserBar.cc) -set(logpot_SRC UserLogPot.cc) -set(disk_SRC UserDisk.cc) -set(halo_SRC UserHalo.cc) -set(mndisk_SRC UserMNdisk.cc) -set(mwgala_SRC UserMW.cc) - -if(ENABLE_CUDA) set(cudatest_SRC UserTestCuda.cc cudaUserTest.cu) - list(APPEND barSRC cudauserBar.cu) + list(APPEND barSRC cudaUserBar.cu) endif() foreach(mlib ${USER_MODULES}) From afd3328f3495d7ef1413a0034ea0da1d62833888 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 1 Apr 2025 17:29:32 -0400 Subject: [PATCH 018/106] Fix up a bunch of typos in UserBar cuda implementation --- src/user/CMakeLists.txt | 2 +- src/user/UserBar.H | 2 +- src/user/cudaUserBar.cu | 30 +++++++++--------------------- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/user/CMakeLists.txt b/src/user/CMakeLists.txt index e89148314..a419b9c64 100644 --- a/src/user/CMakeLists.txt +++ b/src/user/CMakeLists.txt @@ -31,7 +31,7 @@ if(ENABLE_CUDA) # UserLogPot.cc UserMNdisk.cc UserMW.cc UserTest.cc UserTestCuda.cc # PROPERTIES LANGUAGE CUDA) set(cudatest_SRC UserTestCuda.cc cudaUserTest.cu) - list(APPEND barSRC cudaUserBar.cu) + list(APPEND bar_SRC cudaUserBar.cu) endif() foreach(mlib ${USER_MODULES}) diff --git a/src/user/UserBar.H b/src/user/UserBar.H index 2e4a81cd7..6c07f33f0 100644 --- a/src/user/UserBar.H +++ b/src/user/UserBar.H @@ -24,7 +24,7 @@ class UserBar : public ExternalForce { private: - string angm_name, ctr_name; + std::string angm_name, ctr_name; Component *c0, *c1; void determine_acceleration_and_potential(void); diff --git a/src/user/cudaUserBar.cu b/src/user/cudaUserBar.cu index bde7ce726..fc9ac7b88 100644 --- a/src/user/cudaUserBar.cu +++ b/src/user/cudaUserBar.cu @@ -4,7 +4,7 @@ #include #include -#include "UserSat.H" +#include "UserBar.H" // Global device symbols for CUDA kernel // @@ -55,13 +55,13 @@ userBarForceKernel(dArray P, dArray I, cuFP_t pp = (xx*xx - yy*yy)*cos2p + 2.0*xx*yy*sin2p; if (userBarSoft) { - fac = 1.0 + rr/b5; + fac = 1.0 + rr/userBarB5; ffac = -userBarAmp / pow(fac, 6.0); - nn = pp / (b5*rr); + nn = pp / (userBarB5*rr); } else { - fac = 1.0 + pow(rr/b5, 5.0); + fac = 1.0 + pow(rr/userBarB5, 5.0); ffac = -userBarAmp / (fac*fac); - nn = pp * pow(rr/b5, 3.0)/ (b5*b5); + nn = pp * pow(rr/userBarB5, 3.0)/ (userBarB5*userBarB5); } // Add acceleration @@ -80,14 +80,13 @@ userBarForceKernel(dArray P, dArray I, __global__ -void testConstantsUserSat(cuFP_t tnow) +void testConstantsUserBar(cuFP_t tnow) { printf("-------------------------\n"); - printf("---UserSat constants-----\n"); + printf("---UserBar constants-----\n"); printf("-------------------------\n"); printf(" Time = %e\n", tnow ); printf(" Amp = %e\n", userBarAmp ); - printf(" Amp = %e\n", userBarNumFac ); printf(" b5 = %e\n", userBarB5 ); printf(" Center = %e, %e, %e\n", userBarCen[0], userBarCen[1], userBarCen[2] ); @@ -96,26 +95,17 @@ void testConstantsUserSat(cuFP_t tnow) printf("-------------------------\n"); } -void UserBar::determine_acceration_and_potential_cuda() +void UserBar::determine_acceleration_and_potential_cuda() { // Sanity check // int nbodies = cC->Number(); if (nbodies != static_cast(cC->Particles().size())) { - std::cerr << "UserSat: ooops! number=" << nbodies + std::cerr << "UserBar: ooops! number=" << nbodies << " but particle size=" << cC->Particles().size() << endl; nbodies = static_cast(cC->Particles().size()); } - if (nbodies==0) { // Return if there are no particles - if (verbose) { - cout << "Process " << myid << ": in UserBar, nbodies=0" - << " for Component <" << cC->name << "> at T=" << tnow - << endl; - } - return; - } - double amp = afac * numfac * amplitude/fabs(amplitude) * 0.5*(1.0 + erf( (tnow - Ton )/DeltaT )) * 0.5*(1.0 - erf( (tnow - Toff)/DeltaT )) ; @@ -134,8 +124,6 @@ void UserBar::determine_acceration_and_potential_cuda() cuFP_t ctr[3], dtmp; for (int k=0; k<3; k++) ctr[k] = cn[k]; - cuFP_t dtmp; - cuda_safe_call(cudaMemcpyToSymbol(userBarAmp, &(dtmp=amp), sizeof(cuFP_t), size_t(0), cudaMemcpyHostToDevice), __FILE__, __LINE__, "Error copying userBarAmp"); From e85ac14deaf405a53cbe6d22eee5325e05dcd9b8 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 2 Apr 2025 10:40:17 -0400 Subject: [PATCH 019/106] Work-around for deprecation of thrust::unary_function type --- include/cudaParticle.cuH | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/cudaParticle.cuH b/include/cudaParticle.cuH index 8673f0462..5147b01dd 100644 --- a/include/cudaParticle.cuH +++ b/include/cudaParticle.cuH @@ -90,8 +90,12 @@ extern __host__ std::ostream& operator<< (std::ostream& os, const cudaParticle& p); //! Functor for extracting level info from cudaParticle structures +#if CUDART_VERSION < 12040 struct cuPartToLevel : public thrust::unary_function +#else +struct cuPartToLevel +#endif { int _t; @@ -106,8 +110,12 @@ struct cuPartToLevel : }; //! Functor for extracting change level info from cudaParticle structures +#if CUDART_VERSION < 12040 struct cuPartToChange : public thrust::unary_function> +#else +struct cuPartToChange +#endif { __host__ __device__ thrust::pair operator()(const cudaParticle& p) const From 6cfc33e7597a0e43bf086007aa9cd7f03cb37ede Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 7 Apr 2025 13:53:14 -0400 Subject: [PATCH 020/106] Bump devel version to match main v7.8.0 release --- CMakeLists.txt | 2 +- doc/exp.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0528aacbe..ffba0d1b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.25) # Needed for CUDA, MPI, and CTest features project( EXP - VERSION "7.8.0" + VERSION "7.8.1" HOMEPAGE_URL https://github.com/EXP-code/EXP LANGUAGES C CXX Fortran) diff --git a/doc/exp.cfg b/doc/exp.cfg index 1405e49c3..eb9d53c5c 100644 --- a/doc/exp.cfg +++ b/doc/exp.cfg @@ -48,7 +48,7 @@ PROJECT_NAME = EXP # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 7.8.0 +PROJECT_NUMBER = 7.8.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From ec11f5481c3c8e5c4271d6473e6a43ca6c93dbff Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 22 Apr 2025 14:11:22 -0400 Subject: [PATCH 021/106] Expose the pseudo-force infrastructure through pybind11 --- expui/BasisFactory.H | 7 +++- expui/BasisFactory.cc | 16 ++++++-- expui/BiorthBasis.cc | 3 +- pyEXP/BasisWrappers.cc | 89 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/expui/BasisFactory.H b/expui/BasisFactory.H index 19b2fe6e6..2ee5ea584 100644 --- a/expui/BasisFactory.H +++ b/expui/BasisFactory.H @@ -249,8 +249,8 @@ namespace BasisClasses //@{ //! Initialize non-inertial forces - void setNonInertial(int N, Eigen::VectorXd& x, Eigen::MatrixXd& pos); - void setNonInertial(int N, const std::string& orient); + void setNonInertial(const Eigen::VectorXd& x, const Eigen::MatrixXd& pos); + void setNonInertial(const std::string& orient); //@} //! Set the current pseudo acceleration @@ -259,6 +259,9 @@ namespace BasisClasses if (Naccel > 0) pseudo = currentAccel(time); } + //! Returns true if non-intertial forces are active + bool usingNonInertial() { return Naccel > 0; } + //! Get the field label vector std::vector getFieldLabels(void) { return getFieldLabels(coordinates); } diff --git a/expui/BasisFactory.cc b/expui/BasisFactory.cc index 483dcca46..7760eb038 100644 --- a/expui/BasisFactory.cc +++ b/expui/BasisFactory.cc @@ -283,14 +283,22 @@ namespace BasisClasses return makeFromArray(time); } - void Basis::setNonInertial(int N, Eigen::VectorXd& t, Eigen::MatrixXd& pos) + void Basis::setNonInertial(const Eigen::VectorXd& t, const Eigen::MatrixXd& pos) { - Naccel = N; + // Sanity checks + if (t.size() < 1) + throw std::runtime_error("Basis: setNonInertial: no times in time array"); + + if (t.size() != pos.rows()) + throw std::runtime_error("Basis::setNonInertial: size mismatch in time and position arrays"); + + // Set the data + Naccel = t.size(); t_accel = t; p_accel = pos; } - void Basis::setNonInertial(int N, const std::string& orient) + void Basis::setNonInertial(const std::string& orient) { std::ifstream in(orient); @@ -337,7 +345,7 @@ namespace BasisClasses } // Repack data - Naccel = N; + Naccel = times.size(); t_accel.resize(times.size()); p_accel.resize(times.size(), 3); for (int i=0; igetCenter(); + std::vector ctr {0, 0, 0}; + if (basis->usingNonInertial()) ctr = basis->getCenter(); // Get fields // diff --git a/pyEXP/BasisWrappers.cc b/pyEXP/BasisWrappers.cc index bb214cf20..c713d10c4 100644 --- a/pyEXP/BasisWrappers.cc +++ b/pyEXP/BasisWrappers.cc @@ -199,6 +199,18 @@ void BasisFactoryClasses(py::module &m) a fixed potential model. AccelFunc can be inherited by a native Python class and the evalcoefs() may be implemented in Python and passed to IntegrateOrbits in the same way as a native C++ class. + + Non-inertial frames of reference + -------------------------------- + Each component of a multiple component simulation may have its own expansion + center. Orbit integration in the frame of reference of the expansion is + accomplished by defining a moving frame of reference using the setNonInertial() + call with either an array of times (N) and center positions (as an Nx3 array) + or by initializing with an EXP orient file. + + We provide a member function, setNonInertialAccel(t), to estimate the frame + acceleration at a given time. This may be useful for user-defined acceleration + routines. This is automatically called default C++ evalcoefs() routine. )"; using namespace BasisClasses; @@ -945,9 +957,82 @@ void BasisFactoryClasses(py::module &m) ------- list: str list of basis function labels - )" - ); + )" + ) + .def("setNonInertial", + [](BasisClasses::Basis& A, + const Eigen::VectorXd& times, const Eigen::MatrixXd& pos) { + A.setNonInertial(times, pos); + }, + R"( + Initialize for pseudo-force computation with a time series of positions + using (1) a time vector and (2) a center position matrix with rows of three + vectors + + Parameters + ---------- + times : list or numpy.ndarray + list of time points + pos : numpy.ndarray + an array with N rows and 3 columns of center positions + + Returns + ------- + None + + See also + -------- + setNonInertial : set non-inertial data from an Orient file + setNonInertialAccel : set the non-inertial acceration + )", + py::arg("times"), py::arg("pos") + ) + .def("setNonInertial", + [](BasisClasses::Basis& A, const std::string orient) + { + A.setNonInertial(orient); + }, + R"( + Initialize for pseudo-force computation with a time series of positions + using a EXP orient file + + Parameters + ---------- + orient : str + name of the orient file + + Returns + ------- + None + + See also + -------- + setNonInertial : set non-inertial data from a time series of values + setNonInertialAccel : set the non-inertial acceration + )", + py::arg("orient") + ) + .def("setNonInertialAccel", &BasisClasses::Basis::setNonInertialAccel, + R"( + Set the pseudo acceleration for the non-inertial data at a given time + Parameters + ---------- + time : float + evaluation time + + Returns + ------- + None + + See also + -------- + setNonInertial : set non-inertial data from a time series of values + setNonInertial : set non-inertial data from an EXP orient file + )", + py::arg("time") + ); + py::class_, PyBiorthBasis, BasisClasses::Basis> (m, "BiorthBasis") .def(py::init(), From dd1105cb6fb3d48483d83145eef0eebebb7b17b7 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 22 Apr 2025 17:15:28 -0400 Subject: [PATCH 022/106] Fixed the flipped logic of inertial vs non-inertial center subtraction --- expui/BiorthBasis.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 54e08c063..fb0a0dcef 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -3687,8 +3687,8 @@ namespace BasisClasses // Get expansion center // - std::vector ctr {0, 0, 0}; - if (basis->usingNonInertial()) ctr = basis->getCenter(); + auto ctr = basis->getCenter(); + if (basis->usingNonInertial()) ctr = {0, 0, 0}; // Get fields // From 8accceb37d5f08bd8137eaa750f2312bbf1550fd Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 22 Apr 2025 18:32:24 -0400 Subject: [PATCH 023/106] Restore Naccel setting, oops --- expui/BasisFactory.H | 4 ++-- expui/BasisFactory.cc | 8 ++++---- pyEXP/BasisWrappers.cc | 18 +++++++++++------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/expui/BasisFactory.H b/expui/BasisFactory.H index 2ee5ea584..511fc5669 100644 --- a/expui/BasisFactory.H +++ b/expui/BasisFactory.H @@ -249,8 +249,8 @@ namespace BasisClasses //@{ //! Initialize non-inertial forces - void setNonInertial(const Eigen::VectorXd& x, const Eigen::MatrixXd& pos); - void setNonInertial(const std::string& orient); + void setNonInertial(int Naccel, const Eigen::VectorXd& x, const Eigen::MatrixXd& pos); + void setNonInertial(int Naccel, const std::string& orient); //@} //! Set the current pseudo acceleration diff --git a/expui/BasisFactory.cc b/expui/BasisFactory.cc index 7760eb038..088fd8244 100644 --- a/expui/BasisFactory.cc +++ b/expui/BasisFactory.cc @@ -283,7 +283,7 @@ namespace BasisClasses return makeFromArray(time); } - void Basis::setNonInertial(const Eigen::VectorXd& t, const Eigen::MatrixXd& pos) + void Basis::setNonInertial(int N, const Eigen::VectorXd& t, const Eigen::MatrixXd& pos) { // Sanity checks if (t.size() < 1) @@ -293,12 +293,12 @@ namespace BasisClasses throw std::runtime_error("Basis::setNonInertial: size mismatch in time and position arrays"); // Set the data - Naccel = t.size(); + Naccel = N; t_accel = t; p_accel = pos; } - void Basis::setNonInertial(const std::string& orient) + void Basis::setNonInertial(int N, const std::string& orient) { std::ifstream in(orient); @@ -345,7 +345,7 @@ namespace BasisClasses } // Repack data - Naccel = times.size(); + Naccel = N; t_accel.resize(times.size()); p_accel.resize(times.size(), 3); for (int i=0; i Date: Tue, 22 Apr 2025 22:09:03 -0400 Subject: [PATCH 024/106] Put the evaluation point at the center of the interval when possible --- expui/BasisFactory.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/expui/BasisFactory.cc b/expui/BasisFactory.cc index 088fd8244..cedd05fdc 100644 --- a/expui/BasisFactory.cc +++ b/expui/BasisFactory.cc @@ -368,7 +368,10 @@ namespace BasisClasses else { int imin = 0; int imax = std::lower_bound(t_accel.data(), t_accel.data()+t_accel.size(), time) - t_accel.data(); - if (imax > Naccel) imin = imax - Naccel; + + // Get a range around the current time of approx size Naccel + imax = std::min(t_accel.size()-1, imax + Naccel/2); + imin = std::max(imax - Naccel, 0); int num = imax - imin + 1; Eigen::VectorXd t(num); From 47edaf1234abc2d0d76e4426ae5cc84efda38f7f Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 23 Apr 2025 13:06:49 -0400 Subject: [PATCH 025/106] Added a inertial coodindate reset member; fixed an interpolation fence-post error that did not affect the results. --- expui/BasisFactory.H | 8 ++++++++ expui/BiorthBasis.cc | 24 +++++++++++------------- pyEXP/BasisWrappers.cc | 18 ++++++++++++++++++ 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/expui/BasisFactory.H b/expui/BasisFactory.H index 511fc5669..73059a1e6 100644 --- a/expui/BasisFactory.H +++ b/expui/BasisFactory.H @@ -137,6 +137,7 @@ namespace BasisClasses Eigen::Vector3d currentAccel(double time); public: + //! The current pseudo acceleration Eigen::Vector3d pseudo {0, 0, 0}; @@ -262,6 +263,13 @@ namespace BasisClasses //! Returns true if non-intertial forces are active bool usingNonInertial() { return Naccel > 0; } + //! Reset to inertial coordinates + void setInertial() + { + Naccel = 0; + pseudo = {0, 0, 0}; + } + //! Get the field label vector std::vector getFieldLabels(void) { return getFieldLabels(coordinates); } diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index fb0a0dcef..96164d328 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -3727,13 +3727,12 @@ namespace BasisClasses throw std::runtime_error(sout.str()); } - auto it1 = std::lower_bound(times.begin(), times.end(), t); - auto it2 = it1 + 1; + auto it2 = std::lower_bound(times.begin(), times.end(), t); + auto it1 = it2; - if (it2 == times.end()) { - it2--; - it1 = it2 - 1; - } + if (it2 == times.end()) throw std::runtime_error("Basis::AllTimeAccel::evalcoefs: time t=" + std::to_string(t) + " out of bounds"); + else if (it2 == times.begin()) it2++; + else it1--; double a = (*it2 - t)/(*it2 - *it1); double b = (t - *it1)/(*it2 - *it1); @@ -3791,13 +3790,12 @@ namespace BasisClasses throw std::runtime_error(sout.str()); } - auto it1 = std::lower_bound(times.begin(), times.end(), t); - auto it2 = it1 + 1; - - if (it2 == times.end()) { - it2--; - it1 = it2 - 1; - } + auto it2 = std::lower_bound(times.begin(), times.end(), t); + auto it1 = it2; + + if (it2 == times.end()) throw std::runtime_error("Basis::AllTimeAccel::evalcoefs: time t=" + std::to_string(t) + " out of bounds"); + else if (it2 == times.begin()) it2++; + else it1--; double a = (*it2 - t)/(*it2 - *it1); double b = (t - *it1)/(*it2 - *it1); diff --git a/pyEXP/BasisWrappers.cc b/pyEXP/BasisWrappers.cc index 27b3d1528..cef5ac170 100644 --- a/pyEXP/BasisWrappers.cc +++ b/pyEXP/BasisWrappers.cc @@ -959,6 +959,24 @@ void BasisFactoryClasses(py::module &m) list of basis function labels )" ) + .def("setInertial", &BasisClasses::Basis::setInertial, + R"( + Reset to inertial coordinates + + Parameters + ---------- + None + + Returns + ------- + None + + See also + -------- + setNonInertial : set non-inertial data + setNonInertialAccel : set the non-inertial acceration + )" + ) .def("setNonInertial", [](BasisClasses::Basis& A, int N, const Eigen::VectorXd& times, const Eigen::MatrixXd& pos) { From 2cb664a8507b63469495c15ed5043106a9ae5506 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 23 Apr 2025 13:29:37 -0400 Subject: [PATCH 026/106] Comment typo only --- expui/BiorthBasis.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 96164d328..eaac49ab9 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -3697,7 +3697,7 @@ namespace BasisClasses auto v = basis->getFields(ps(n, 0) - ctr[0], ps(n, 1) - ctr[1], ps(n, 2) - ctr[2]); - // First 6 fields are density and potential, follewed by acceleration + // First 6 fields are density and potential, followed by acceleration for (int k=0; k<3; k++) accel(n, k) += v[6+k] - basis->pseudo(k); } From f792b9ca6133b2f6cb092b43c806269ec245fd49 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 23 Apr 2025 16:33:34 -0400 Subject: [PATCH 027/106] Add a bit of tolerance to the center array grid interpolation before flagging out of bounds --- expui/BasisFactory.cc | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/expui/BasisFactory.cc b/expui/BasisFactory.cc index cedd05fdc..b410c7861 100644 --- a/expui/BasisFactory.cc +++ b/expui/BasisFactory.cc @@ -359,18 +359,30 @@ namespace BasisClasses { Eigen::Vector3d ret; - if (time < t_accel(0) || time > t_accel(t_accel.size()-1)) { - std::cout << "ERROR: " << time << " is outside of range of the non-inertial DB" - << std::endl; - ret.setZero(); - return ret; + auto n = t_accel.size(); + + // Allow a little bit of buffer in the allowable on-grid range but + // otherwise force termination + // + if ( time < t_accel(0 ) - 0.5*(t_accel(1 ) - t_accel(0 )) || + time > t_accel(n-1) + 0.5*(t_accel(n-1) - t_accel(n-2)) ) { + + std::ostringstream msg; + msg << "Basis::currentAccel: " << time + << " is outside the range of the non-inertial DB [" + << t_accel(0) << ", " << t_accel(n-1) << "]"; + + throw std::runtime_error(msg.str()); } + // Do the quadratic interpolation + // else { int imin = 0; - int imax = std::lower_bound(t_accel.data(), t_accel.data()+t_accel.size(), time) - t_accel.data(); + int imax = std::lower_bound(t_accel.data(), t_accel.data()+n, time) - t_accel.data(); // Get a range around the current time of approx size Naccel - imax = std::min(t_accel.size()-1, imax + Naccel/2); + // + imax = std::min(n-1, imax + Naccel/2); imin = std::max(imax - Naccel, 0); int num = imax - imin + 1; From c3a227589db6bf5c29dace9d9626b84f9814be22 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 23 Apr 2025 16:34:56 -0400 Subject: [PATCH 028/106] Assign passed coefficients as the coefret internal basis instance so this can be queried by clients; added pseudo accel debug output for testing --- expui/BiorthBasis.cc | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index eaac49ab9..e87c172c0 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -396,6 +396,10 @@ namespace BasisClasses CoefClasses::SphStruct* cf = dynamic_cast(coef.get()); + // Cache the current coefficient structure + // + coefret = coef; + // Assign internal coefficient table (doubles) from the complex struct // for (int l=0, L0=0, L1=0; l<=lmax; l++) { @@ -1474,6 +1478,10 @@ namespace BasisClasses CoefClasses::CylStruct* cf = dynamic_cast(coef.get()); + // Cache the current coefficient structure + // + coefret = coef; + for (int m=0; m<=mmax; m++) { // Set to zero on m=0 call only--------+ sl->set_coefs(m, (*cf->coefs).row(m).real(), (*cf->coefs).row(m).imag(), m==0); } @@ -1771,6 +1779,10 @@ namespace BasisClasses CoefClasses::CylStruct* cf = dynamic_cast(coef.get()); auto & cc = *cf->coefs; + // Cache the current coefficient structure + // + coefret = coef; + // Assign internal coefficient table (doubles) from the complex struct // for (int m=0, m0=0; m<=mmax; m++) { @@ -2474,6 +2486,10 @@ namespace BasisClasses CoefClasses::CylStruct* cf = dynamic_cast(coef.get()); auto & cc = *cf->coefs; + // Cache the cuurent coefficient structure + // + coefret = coef; + // Assign internal coefficient table (doubles) from the complex struct // for (int m=0, m0=0; m<=mmax; m++) { @@ -2874,6 +2890,10 @@ namespace BasisClasses auto cf = dynamic_cast(coef.get()); expcoef = *cf->coefs; + // Cache the current coefficient structure + // + coefret = coef; + coefctr = {0.0, 0.0, 0.0}; } @@ -3306,6 +3326,10 @@ namespace BasisClasses auto cf = dynamic_cast(coef.get()); expcoef = *cf->coefs; + // Cache the cuurent coefficient structure + // + coefret = coef; + coefctr = {0.0, 0.0, 0.0}; } @@ -3701,6 +3725,30 @@ namespace BasisClasses for (int k=0; k<3; k++) accel(n, k) += v[6+k] - basis->pseudo(k); } + // true for deep debugging + // | + // v + if (true and basis->usingNonInertial()) { + + auto coefs = basis->getCoefficients(); + auto time = coefs->time; + auto ctr = coefs->ctr; + + std::ofstream tmp; + if (time <= 0.0) tmp.open("pseudo.dat"); + else tmp.open("pseudo.dat", ios::app); + + if (tmp) + tmp << std::setw(16) << std::setprecision(5) << time + << std::setw(16) << std::setprecision(5) << ctr[0] + << std::setw(16) << std::setprecision(5) << ctr[1] + << std::setw(16) << std::setprecision(5) << ctr[2] + << std::setw(16) << std::setprecision(5) << basis->pseudo(0) + << std::setw(16) << std::setprecision(5) << basis->pseudo(1) + << std::setw(16) << std::setprecision(5) << basis->pseudo(2) + << std::endl; + } + return accel; } From c74d5b3d580b4b68067c8842ab549fec51b983f0 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 23 Apr 2025 20:47:40 -0400 Subject: [PATCH 029/106] Some cleanup; turn of RK4 test and pseudo force test output --- expui/BiorthBasis.cc | 104 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 86 insertions(+), 18 deletions(-) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index e87c172c0..1972ad9bd 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -635,7 +635,7 @@ namespace BasisClasses Spherical::crt_eval (double x, double y, double z) { - double R = sqrt(x*x + y*y); + double R = sqrt(x*x + y*y) + 1.0e-18; double phi = atan2(y, x); auto v = cyl_eval(R, z, phi); @@ -3728,7 +3728,7 @@ namespace BasisClasses // true for deep debugging // | // v - if (true and basis->usingNonInertial()) { + if (false and basis->usingNonInertial()) { auto coefs = basis->getCoefficients(); auto time = coefs->time; @@ -3890,25 +3890,93 @@ namespace BasisClasses { int rows = ps.rows(); - // Drift 1/2 - for (int n=0; n kf(4); + for (int i=0; i<4; i++) kf[i].resize(rows, 6); + + // Step 1 + // + accel.setZero(); + for (auto mod : bfe) F(t, ps, accel, mod); + for (int n=0; n(t+h, ps); } From e4a977c215ca93b435a02ce8008c95cfc9bdd7bc Mon Sep 17 00:00:00 2001 From: Michael Petersen Date: Thu, 24 Apr 2025 10:54:52 +0100 Subject: [PATCH 030/106] Apply suggestions from code review Typos in docstrings only. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- expui/BasisFactory.H | 2 +- expui/BiorthBasis.cc | 3 +-- pyEXP/BasisWrappers.cc | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/expui/BasisFactory.H b/expui/BasisFactory.H index 73059a1e6..1598f3f7d 100644 --- a/expui/BasisFactory.H +++ b/expui/BasisFactory.H @@ -260,7 +260,7 @@ namespace BasisClasses if (Naccel > 0) pseudo = currentAccel(time); } - //! Returns true if non-intertial forces are active + //! Returns true if non-inertial forces are active bool usingNonInertial() { return Naccel > 0; } //! Reset to inertial coordinates diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 1972ad9bd..c0cd6e9e5 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -2486,8 +2486,7 @@ namespace BasisClasses CoefClasses::CylStruct* cf = dynamic_cast(coef.get()); auto & cc = *cf->coefs; - // Cache the cuurent coefficient structure - // + // Cache the current coefficient structure coefret = coef; // Assign internal coefficient table (doubles) from the complex struct diff --git a/pyEXP/BasisWrappers.cc b/pyEXP/BasisWrappers.cc index cef5ac170..fb1332a11 100644 --- a/pyEXP/BasisWrappers.cc +++ b/pyEXP/BasisWrappers.cc @@ -974,7 +974,7 @@ void BasisFactoryClasses(py::module &m) See also -------- setNonInertial : set non-inertial data - setNonInertialAccel : set the non-inertial acceration + setNonInertialAccel : set the non-inertial acceleration )" ) .def("setNonInertial", From 35d26a6ae8c78faff3e6290ddf907f5a4ebedd53 Mon Sep 17 00:00:00 2001 From: michael-petersen Date: Thu, 24 Apr 2025 21:13:36 +0100 Subject: [PATCH 031/106] Return Version tuple for compatibility checking --- pyEXP/UtilWrappers.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyEXP/UtilWrappers.cc b/pyEXP/UtilWrappers.cc index 119005c84..04a80ae01 100644 --- a/pyEXP/UtilWrappers.cc +++ b/pyEXP/UtilWrappers.cc @@ -155,4 +155,22 @@ void UtilityClasses(py::module &m) { Report on the version and git commit. This is the same version information reported by the EXP N-body code.)"); + m.def("Version", + []() { + std::string version_str = VERSION; + std::istringstream iss(version_str); + std::string token; + int parts[3] = {0, 0, 0}; + int i = 0; + + while (std::getline(iss, token, '.') && i < 3) { + parts[i++] = std::stoi(token); + } + + return std::make_tuple(parts[0], parts[1], parts[2]); + }, + R"( + Return the version. + + This is the same version information reported by the EXP N-body code.)"); } From 1cb6c2de74bab092744a819762d17ebe3ccd547f Mon Sep 17 00:00:00 2001 From: michael-petersen Date: Thu, 24 Apr 2025 21:19:47 +0100 Subject: [PATCH 032/106] version bump [no CI] --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dbf34eaf7..9f5742909 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.25) # Needed for CUDA, MPI, and CTest features project( EXP - VERSION "7.8.2" + VERSION "7.8.3" HOMEPAGE_URL https://github.com/EXP-code/EXP LANGUAGES C CXX Fortran) From 6d817dabbaae623d57caca99170839c674bad7de Mon Sep 17 00:00:00 2001 From: michael-petersen Date: Wed, 30 Apr 2025 11:18:13 -0400 Subject: [PATCH 033/106] proposed promotion of totVar and totPow flags --- expui/expMSSA.H | 4 ++-- expui/expMSSA.cc | 23 ++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/expui/expMSSA.H b/expui/expMSSA.H index 6e3563f4c..63cc27b35 100644 --- a/expui/expMSSA.H +++ b/expui/expMSSA.H @@ -84,8 +84,8 @@ namespace MSSA //! Number of components in the reconstruction int ncomp; - //! Normalization values - double totVar, totPow; + //! Normalization options + bool totVar, totPow; //! Toggle for detrending bool useMean; diff --git a/expui/expMSSA.cc b/expui/expMSSA.cc index 176ae7715..f63f2d063 100644 --- a/expui/expMSSA.cc +++ b/expui/expMSSA.cc @@ -1522,8 +1522,8 @@ namespace MSSA { HighFive::Group recon = file.createGroup("reconstruction"); recon.createAttribute ("ncomp", HighFive::DataSpace::From(ncomp) ).write(ncomp); - recon.createAttribute("totVar", HighFive::DataSpace::From(totVar)).write(totVar); - recon.createAttribute("totPow", HighFive::DataSpace::From(totVar)).write(totPow); + recon.createAttribute("totVar", HighFive::DataSpace::From(totVar)).write(totVar); + recon.createAttribute("totPow", HighFive::DataSpace::From(totVar)).write(totPow); for (int n=0; n(); + if (params["totPow"]) totPow = params["totPow"].as(); + + if (totPow==true) { + if (totVar==true) { + std::cerr << "expMSSA: both totVar and totPow are set to true." + << "Using totPow." << std::endl; + totVar = false; + } type = TrendType::totPow; - else + } else if (totVar==true) { + type = TrendType::totVar; + } else { + // if nothing set go default type = TrendType::perChannel; - + } // Set the SVD strategy for mSSA // From 5f756a067a694f2574c411dac382650b9d092bef Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Mon, 5 May 2025 10:24:43 -0400 Subject: [PATCH 034/106] Update EmpCylSL.cc Added spaces for readability --- exputil/EmpCylSL.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exputil/EmpCylSL.cc b/exputil/EmpCylSL.cc index 71135d576..1e631c7a9 100644 --- a/exputil/EmpCylSL.cc +++ b/exputil/EmpCylSL.cc @@ -267,7 +267,7 @@ EmpCylSL::EmpCylSL(int mlim, std::string cachename) if (myid==0) std::cout << "---- EmpCylSL::ReadH5Cache: " << "using a workaround for a HighFive HDF5 wrapper bug. " - << "this workaround will be removed in EXP 7.9.0. " + << "this workaround will be removed in EXP 7.9.0. " << "Please consider rebuilding your cache if possible!" << std::endl; } From 38bd60efb5aee638e82b2cfc5c4ceb4a44863a59 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 5 May 2025 11:36:41 -0400 Subject: [PATCH 035/106] Version bump: devel is now 7.9.0 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f5742909..3a855b74f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.25) # Needed for CUDA, MPI, and CTest features project( EXP - VERSION "7.8.3" + VERSION "7.9.0" HOMEPAGE_URL https://github.com/EXP-code/EXP LANGUAGES C CXX Fortran) From 41b389a5ee238d2dcaa30a777c912eab6864f8e9 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 19 May 2025 14:43:51 -0400 Subject: [PATCH 036/106] Add the sech2 parameter for Basis::Cylindrical and Cylinder use hcyl as sech^2(h/(2*hcyl)) for disk conditioning rather than the current default sech^2(z/h) --- expui/BiorthBasis.H | 2 +- expui/BiorthBasis.cc | 16 +++++++++++----- src/Cylinder.H | 4 +++- src/Cylinder.cc | 9 +++++++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/expui/BiorthBasis.H b/expui/BiorthBasis.H index 045e209e7..5e437e237 100644 --- a/expui/BiorthBasis.H +++ b/expui/BiorthBasis.H @@ -718,7 +718,7 @@ namespace BasisClasses int rnum, pnum, tnum; double rmin, rmax, rcylmin, rcylmax; double acyl, hcyl; - bool expcond, logarithmic, density, EVEN_M; + bool expcond, logarithmic, density, EVEN_M, sech2 = false; std::vector potd, dpot, dpt2, dend; std::vector legs, dlegs, d2legs; diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 93f73a336..46b8702e4 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -874,6 +874,7 @@ namespace BasisClasses "rcylmax", "acyl", "hcyl", + "sech2", "snr", "evcut", "nmaxfid", @@ -980,6 +981,8 @@ namespace BasisClasses double w1 = 1.0/(1.0+dweight); double w2 = dweight/(1.0+dweight); + if (sech2) { h1 *= 0.5; h2 *= 0.5; } + double f1 = cosh(z/h1); double f2 = cosh(z/h2); @@ -991,13 +994,14 @@ namespace BasisClasses case DiskType::diskbulge: { - double f = cosh(z/hcyl); + double h = sech2 ? 0.5*hcyl : hcyl; + double f = cosh(z/h); double rr = pow(pow(R, 2) + pow(z,2), 0.5); double w1 = Mfac; double w2 = 1.0 - Mfac; double as = HERNA; - ans = w1*exp(-R/acyl)/(4.0*M_PI*acyl*acyl*hcyl*f*f) + + ans = w1*exp(-R/acyl)/(4.0*M_PI*acyl*acyl*h*f*f) + w2*pow(as, 4)/(2.0*M_PI*rr)*pow(rr+as,-3.0) ; } break; @@ -1007,8 +1011,9 @@ namespace BasisClasses case DiskType::exponential: default: { - double f = cosh(z/hcyl); - ans = exp(-R/acyl)/(4.0*M_PI*acyl*acyl*hcyl*f*f); + double h = sech2 ? 0.5*hcyl : hcyl; + double f = cosh(z/h); + ans = exp(-R/acyl)/(4.0*M_PI*acyl*acyl*h*f*f); } break; } @@ -1130,6 +1135,7 @@ namespace BasisClasses if (conf["acyl" ]) acyl = conf["acyl" ].as(); if (conf["hcyl" ]) hcyl = conf["hcyl" ].as(); + if (conf["sech2" ]) sech2 = conf["sech2" ].as(); if (conf["lmaxfid" ]) lmaxfid = conf["lmaxfid" ].as(); if (conf["nmaxfid" ]) nmaxfid = conf["nmaxfid" ].as(); if (conf["nmax" ]) nmax = conf["nmax" ].as(); @@ -1325,7 +1331,7 @@ namespace BasisClasses // The scale in EmpCylSL is assumed to be 1 so we compute the // height relative to the length // - double H = hcyl/acyl; + double H = sech2 ? 0.5*hcyl/acyl : hcyl/acyl; // The model instance (you can add others in DiskModels.H). // It's MN or Exponential if not MN. diff --git a/src/Cylinder.H b/src/Cylinder.H index 0faf7ddaa..071420faa 100644 --- a/src/Cylinder.H +++ b/src/Cylinder.H @@ -35,6 +35,8 @@ class MixtureBasis; @param hcyl is the scale height + @param sech2 if true, use hcyl as sech^2(z/(2*hcyl)) + @param lmax is the maximum spherical harmonic index for EOF construction @param mmax is the maximum azimuthal order for the resulting basis @@ -110,7 +112,7 @@ class Cylinder : public Basis { private: - bool precond, EVEN_M, subsamp; + bool precond, EVEN_M, subsamp, sech2 = false; int rnum, pnum, tnum; double ashift; unsigned int vflag; diff --git a/src/Cylinder.cc b/src/Cylinder.cc index af8103e0d..45612c71c 100644 --- a/src/Cylinder.cc +++ b/src/Cylinder.cc @@ -29,6 +29,7 @@ Cylinder::valid_keys = { "rcylmax", "acyl", "hcyl", + "sech2", "hexp", "snr", "evcut", @@ -271,9 +272,10 @@ Cylinder::Cylinder(Component* c0, const YAML::Node& conf, MixtureBasis *m) : auto DiskDens = [&](double R, double z, double phi) { if (dfunc) return (*dfunc)(R, z, phi); - double f = exp(-fabs(z)/hcyl); // Overflow prevention + double h = sech2 ? 0.5*hcyl : hcyl; + double f = exp(-fabs(z)/h); // Overflow prevention double s = 2.0*f/(1.0 + f*f); // in sech computation - return exp(-R/acyl)*s*s/(4.0*M_PI*acyl*acyl*hcyl); + return exp(-R/acyl)*s*s/(4.0*M_PI*acyl*acyl*h); }; // The conditioning function for the EOF with an optional shift @@ -339,6 +341,7 @@ Cylinder::Cylinder(Component* c0, const YAML::Node& conf, MixtureBasis *m) : << std::endl << sep << "npca0=" << npca0 << std::endl << sep << "pcadiag=" << pcadiag << std::endl << sep << "cachename=" << cachename + << std::endl << sep << "sech2=" << std::boolalpha << sech2 << std::endl << sep << "selfgrav=" << std::boolalpha << self_consistent << std::endl << sep << "logarithmic=" << logarithmic << std::endl << sep << "vflag=" << vflag @@ -367,6 +370,7 @@ Cylinder::Cylinder(Component* c0, const YAML::Node& conf, MixtureBasis *m) : << std::endl << sep << "npca0=" << npca0 << std::endl << sep << "pcadiag=" << pcadiag << std::endl << sep << "cachename=" << cachename + << std::endl << sep << "sech2=" << std::boolalpha << sech2 << std::endl << sep << "selfgrav=" << std::boolalpha << self_consistent << std::endl << sep << "logarithmic=" << logarithmic << std::endl << sep << "vflag=" << vflag @@ -414,6 +418,7 @@ void Cylinder::initialize() if (conf["acyl" ]) acyl = conf["acyl" ].as(); if (conf["hcyl" ]) hcyl = conf["hcyl" ].as(); + if (conf["sech2" ]) sech2 = conf["sech2" ].as(); if (conf["hexp" ]) hexp = conf["hexp" ].as(); if (conf["snr" ]) snr = conf["snr" ].as(); if (conf["evcut" ]) rem = conf["evcut" ].as(); From b7c7f057c3f7c38ed6560d2ecaf353e92d396f95 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 19 May 2025 15:32:52 -0400 Subject: [PATCH 037/106] Add explanation and deprecation warnings for sech2 --- expui/BiorthBasis.cc | 14 +++++++++++++- src/Cylinder.cc | 8 ++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 46b8702e4..2beb4529a 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -1306,8 +1306,20 @@ namespace BasisClasses try { // Check for map entry, will through if the DTYPE = dtlookup.at(dtype); // key is not in the map. - if (myid==0) // Report DiskType + if (myid==0) { // Report DiskType std::cout << "---- DiskType is <" << dtype << ">" << std::endl; + + if (not sech2) { + switch (DTYPE) { + case DiskType::doubleexpon: + case DiskType::exponential: + case DiskType::diskbulge: + std::cout << "---- pyEXP uses sech^2(z/h) rather than the more common sech^2(z/(2h))" << std::endl + << "---- Use the 'sech2: true' in your YAML config to use sech^2(z/(2h))" << std::endl + << "---- pyEXP will assume sech^2(z/(2h)) by default in v 7.9.0 and later" << std::endl; + } + } + } } catch (const std::out_of_range& err) { if (myid==0) { diff --git a/src/Cylinder.cc b/src/Cylinder.cc index 45612c71c..54cca39b3 100644 --- a/src/Cylinder.cc +++ b/src/Cylinder.cc @@ -458,6 +458,14 @@ void Cylinder::initialize() if (conf["cmapz" ]) cmapZ = conf["cmapz" ].as(); if (conf["vflag" ]) vflag = conf["vflag" ].as(); + // Deprecation warning + if (not sech2 and not conf["pyname"]) { + if (myid==0) + std::cout << "---- Cylinder uses sech^2(z/h) rather than the more common sech^2(z/(2h))" << std::endl + << "---- Use the 'sech2: true' in your YAML config to use sech^2(z/(2h))" << std::endl + << "---- Cylinder will assume sech^2(z/(2h)) by default in v 7.9.0 and later" << std::endl; + } + // Deprecation warning if (conf["expcond"]) { if (myid==0) From c5823f67db13c6ffdfdd8d33630eb7a4d2043d65 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 21 May 2025 11:42:51 -0400 Subject: [PATCH 038/106] Initial draft commit; still need to test and fix a few stomps --- expui/BiorthBasis.H | 29 ++++ expui/BiorthBasis.cc | 298 +++++++++++++++++++++++++++++++++++++---- pyEXP/BasisWrappers.cc | 26 +++- 3 files changed, 329 insertions(+), 24 deletions(-) diff --git a/expui/BiorthBasis.H b/expui/BiorthBasis.H index 045e209e7..0fd25c090 100644 --- a/expui/BiorthBasis.H +++ b/expui/BiorthBasis.H @@ -163,6 +163,11 @@ namespace BasisClasses //! Clear the particle selector callback void clrSelector() { ftor = nullptr; } + + //! Evaluate acceleration in Cartesian coordinates in centered + //! coordinate system + virtual std::vector + getAccel(double x, double y, double z) = 0; }; /** @@ -307,6 +312,9 @@ namespace BasisClasses return ret; } + //! Evaluate acceleration in Cartesian coordinates in centered + //! coordinate system + virtual std::vector getAccel(double x, double y, double z); }; /** @@ -542,6 +550,10 @@ namespace BasisClasses return ret; } + //! Evaluate acceleration in Cartesian coordinates in centered + //! coordinate system + std::vector getAccel(double x, double y, double z); + }; /** @@ -693,6 +705,10 @@ namespace BasisClasses return ret; } + //! Evaluate acceleration in Cartesian coordinates in centered + //! coordinate system + std::vector getAccel(double x, double y, double z); + }; /** @@ -845,6 +861,11 @@ namespace BasisClasses std::cout << "---- Cylindrical::orthoTest: worst=" << worst << std::endl; return ret; } + + //! Evaluate acceleration in Cartesian coordinates in centered + //! coordinate system + std::vector getAccel(double x, double y, double z); + }; /** @@ -991,6 +1012,10 @@ namespace BasisClasses return true; } + //! Evaluate acceleration in Cartesian coordinates in centered + //! coordinate system + std::vector getAccel(double x, double y, double z); + }; /** @@ -1113,6 +1138,10 @@ namespace BasisClasses return true; } + //! Evaluate acceleration in Cartesian coordinates in centered + //! coordinate system + std::vector getAccel(double x, double y, double z); + }; diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 5448c6468..fc728f242 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -506,8 +506,13 @@ namespace BasisClasses } std::vector - Spherical::sph_eval(double r, double costh, double phi) + Spherical::getAccel(double x, double y, double z) { + double R = sqrt(x*x + y*y); + double r = sqrt(R*R + z*z); + double costh = z/r; + double phi = atan2(y, x); + // Get thread id int tid = omp_get_thread_num(); @@ -516,20 +521,17 @@ namespace BasisClasses fac1 = factorial(0, 0); - get_dens (dend[tid], r/scale); get_pot (potd[tid], r/scale); get_force(dpot[tid], r/scale); legendre_R(lmax, costh, legs[tid], dlegs[tid]); - double den0, pot0, potr; + double pot0, potr; if (NO_L0) { - den0 = 0.0; pot0 = 0.0; potr = 0.0; } else { - den0 = fac1 * expcoef.row(0).dot(dend[tid].row(0)); pot0 = fac1 * expcoef.row(0).dot(potd[tid].row(0)); potr = fac1 * expcoef.row(0).dot(dpot[tid].row(0)); } @@ -563,7 +565,6 @@ namespace BasisClasses sumD += expcoef(loffset+moffset, n) * dpot[tid](l, n); } - den1 += fac1*legs[tid] (l, m) * sumR; pot1 += fac1*legs[tid] (l, m) * sumP; potr += fac1*legs[tid] (l, m) * sumD; pott += fac1*dlegs[tid](l, m) * sumP; @@ -577,15 +578,12 @@ namespace BasisClasses double sumR0=0.0, sumP0=0.0, sumD0=0.0; double sumR1=0.0, sumP1=0.0, sumD1=0.0; for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { - sumR0 += expcoef(loffset+moffset+0, n) * dend[tid](l, n); sumP0 += expcoef(loffset+moffset+0, n) * potd[tid](l, n); sumD0 += expcoef(loffset+moffset+0, n) * dpot[tid](l, n); - sumR1 += expcoef(loffset+moffset+1, n) * dend[tid](l, n); sumP1 += expcoef(loffset+moffset+1, n) * potd[tid](l, n); sumD1 += expcoef(loffset+moffset+1, n) * dpot[tid](l, n); } - den1 += fac1 * legs[tid] (l, m) * ( sumR0*cosm + sumR1*sinm ); pot1 += fac1 * legs[tid] (l, m) * ( sumP0*cosm + sumP1*sinm ); potr += fac1 * legs[tid] (l, m) * ( sumD0*cosm + sumD1*sinm ); pott += fac1 * dlegs[tid](l, m) * ( sumP0*cosm + sumP1*sinm ); @@ -596,22 +594,21 @@ namespace BasisClasses } } - double densfac = 1.0/(scale*scale*scale) * 0.25/M_PI; double potlfac = 1.0/scale; - return - {den0 * densfac, // 0 - den1 * densfac, // 1 - (den0 + den1) * densfac, // 2 - pot0 * potlfac, // 3 - pot1 * potlfac, // 4 - (pot0 + pot1) * potlfac, // 5 - potr * (-potlfac)/scale, // 6 - pott * (-potlfac), // 7 - potp * (-potlfac)}; // 8 - // ^ - // | + potr *= (-potlfac)/scale; + pott *= (-potlfac); + potp *= (-potlfac); + + double potR = potr*sinth + pott*costh; + double potz = potr*costh - pott*sinth; + + double tpotx = potR*x/R - potp*y/R ; + double tpoty = potR*y/R + potp*x/R ; + // Return force not potential gradient + // + return {tpotx, tpoty, potz}; } @@ -646,7 +643,6 @@ namespace BasisClasses return {v[0], v[1], v[2], v[3], v[4], v[5], tpotx, tpoty, v[7]}; } - Spherical::BasisArray SphericalSL::getBasis (double logxmin, double logxmax, int numgrid) { @@ -1415,6 +1411,24 @@ namespace BasisClasses tpotl0, tpotl - tpotl0, tpotl, tpotx, tpoty, tpotz}; } + // Evaluate in cartesian coordinates + std::vector Cylindrical::getAccel(double x, double y, double z) + { + double R = sqrt(x*x + y*y); + double phi = atan2(y, x); + + double tdens0, tdens, tpotl0, tpotl, tpotR, tpotz, tpotp; + + sl->accumulated_eval(R, z, phi, tpotl0, tpotl, tpotR, tpotz, tpotp); + + tdens = sl->accumulated_dens_eval(R, z, phi, tdens0); + + double tpotx = tpotR*x/R - tpotp*y/R ; + double tpoty = tpotR*y/R + tpotp*x/R ; + + return {tpotx, tpoty, tpotz}; + } + // Evaluate in cylindrical coordinates std::vector Cylindrical::cyl_eval(double R, double z, double phi) { @@ -1979,6 +1993,89 @@ namespace BasisClasses return {den0, den1, den0+den1, pot0, pot1, pot0+pot1, rpot, zpot, ppot}; } + std::vectorFlatDisk::getAccel(double R, double z, double phi) + { + // Get thread id + int tid = omp_get_thread_num(); + + // Fixed values + constexpr double norm0 = 0.5*M_2_SQRTPI/M_SQRT2; + constexpr double norm1 = 0.5*M_2_SQRTPI; + + double rpot=0, zpot=0, ppot=0; + + // Off grid evaluation + if (R>ortho->getRtable() or fabs(z)>ortho->getRtable()) { + double r2 = R*R + z*z; + double r = sqrt(r2); + + rpot = -totalMass*R/(r*r2 + 10.0*std::numeric_limits::min()); + zpot = -totalMass*z/(r*r2 + 10.0*std::numeric_limits::min()); + + return {rpot, zpot, ppot}; + } + + // Get the basis fields + // + ortho->get_pot (potd[tid], R, z); + ortho->get_rforce (potR[tid], R, z); + ortho->get_zforce (potZ[tid], R, z); + + // m loop + // + for (int m=0, moffset=0; m<=mmax; m++) { + + if (m==0 and NO_M0) { moffset++; continue; } + if (m==1 and NO_M1) { moffset += 2; continue; } + if (EVEN_M and m/2*2 != m) { moffset += 2; continue; } + if (m>0 and M0_only) break; + + if (m==0) { + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + rpot += expcoef(0, n) * potR[tid](0, n) * norm0; + zpot += expcoef(0, n) * potZ[tid](0, n) * norm0; + } + + moffset++; + } else { + double cosm = cos(phi*m), sinm = sin(phi*m); + double vc, vs; + + vc = vs = 0.0; + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + vc += expcoef(moffset+0, n) * potd[tid](m, n); + vs += expcoef(moffset+1, n) * potd[tid](m, n); + } + + ppot += (-vc*sinm + vs*cosm) * m * norm1; + + vc = vs = 0.0; + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + vc += expcoef(moffset+0, n) * potR[tid](m, n); + vs += expcoef(moffset+1, n) * potR[tid](m, n); + } + + rpot += (vc*cosm + vs*sinm) * norm1; + + vc = vs = 0.0; + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + vc += expcoef(moffset+0, n) * potZ[tid](m, n); + vs += expcoef(moffset+1, n) * potZ[tid](m, n); + } + + zpot += (vc*cosm + vs*sinm) * norm1; + + moffset +=2; + } + } + + rpot *= -1.0; + zpot *= -1.0; + ppot *= -1.0; + + return {rpot, zpot, ppot}; + } + std::vector FlatDisk::sph_eval(double r, double costh, double phi) { @@ -2660,6 +2757,71 @@ namespace BasisClasses } + std::vectorCBDisk::getAccel(double x, double y, double z) + { + // Get thread id + int tid = omp_get_thread_num(); + + // Fixed values + constexpr double norm0 = 1.0; + constexpr double norm1 = M_SQRT2; + + double R = std::sqrt(x*x + y*y); + double phi = std::atan2(y, x); + + double rpot=0, zpot=0, ppot=0; + + // Get the basis fields + // + get_pot (potd[tid], R); + get_force (potR[tid], R); + + // m loop + // + for (int m=0, moffset=0; m<=mmax; m++) { + + if (m==0 and NO_M0) { moffset++; continue; } + if (m==1 and NO_M1) { moffset += 2; continue; } + if (EVEN_M and m/2*2 != m) { moffset += 2; continue; } + if (m>0 and M0_only) break; + + if (m==0) { + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + rpot += expcoef(0, n) * potR[tid](0, n) * norm0; + } + + moffset++; + } else { + double cosm = cos(phi*m), sinm = sin(phi*m); + double vc, vs; + + vc = vs = 0.0; + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + vc += expcoef(moffset+0, n) * potd[tid](m, n); + vs += expcoef(moffset+1, n) * potd[tid](m, n); + } + + ppot += (-vc*sinm + vs*cosm) * m * norm1; + + vc = vs = 0.0; + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + vc += expcoef(moffset+0, n) * potR[tid](m, n); + vs += expcoef(moffset+1, n) * potR[tid](m, n); + } + + rpot += (vc*cosm + vs*sinm) * norm1; + + moffset +=2; + } + } + + rpot *= -1.0; + ppot *= -1.0; + + return {rpot, zpot, ppot}; + } + + std::vector CBDisk::sph_eval(double r, double costh, double phi) { // Cylindrical coords @@ -3063,6 +3225,82 @@ namespace BasisClasses } + std::vector + Slab::getAccel(double x, double y, double z) + { + // Loop indices + // + int ix, iy, iz; + + // Working values + // + std::complex facx, facy, fac, facf, facd; + + // Return values + // + std::complex accx(0.0), accy(0.0), accz(0.0); + + // Recursion multipliers + // + std::complex stepx = exp(kfac*x); + std::complex stepy = exp(kfac*y); + + // Initial values (note sign change) + // + std::complex startx = exp(-static_cast(nmaxx)*kfac*x); + std::complex starty = exp(-static_cast(nmaxy)*kfac*y); + + Eigen::VectorXd vpot(nmaxz), vfrc(nmaxz), vden(nmaxz); + + for (facx=startx, ix=0; ix nmaxx) { + std::cerr << "Out of bounds: ii=" << ii << std::endl; + } + if (iiy > nmaxy) { + std::cerr << "Out of bounds: jj=" << jj << std::endl; + } + + if (iix>=iiy) { + ortho->get_force(vfrc, z, iix, iiy); + } + else { + ortho->get_force(vfrc, z, iiy, iix); + } + + + for (int iz=0; iz(ii)*fac; + accy += -kfac*static_cast(jj)*fac; + accz += -facf; + + } + } + } + + return {accx.real(), accy.real(), accz.real()}; + } + std::vector Slab::crt_eval(double x, double y, double z) { @@ -3425,6 +3663,20 @@ namespace BasisClasses return {0, den1, den1, 0, pot1, pot1, frcx, frcy, frcz}; } + std::vector Cube::getAccel(double x, double y, double z) + { + // Get thread id + int tid = omp_get_thread_num(); + + // Position vector + Eigen::Vector3d pos {x, y, z}; + + // Get the basis fields + auto frc = ortho->get_force(expcoef, pos); + + return {-frc(0).real(), -frc(1).real(), -frc(2).real()}; + } + std::vector Cube::cyl_eval(double R, double z, double phi) { // Get thread id diff --git a/pyEXP/BasisWrappers.cc b/pyEXP/BasisWrappers.cc index fb1332a11..ced28b0d1 100644 --- a/pyEXP/BasisWrappers.cc +++ b/pyEXP/BasisWrappers.cc @@ -1217,7 +1217,31 @@ void BasisFactoryClasses(py::module &m) See also -------- getFieldsCoefs : get fields for each coefficient set - __call__ : same getFields() but provides field labels in a tuple + __call__ : same as getFields() but provides field labels in a tuple + )", + py::arg("x"), py::arg("y"), py::arg("z")) + .def("getAccel", &BasisClasses::BiorthBasis::getAccel, + R"( + Return the acceleration for a given cartesian position + + Parameters + ---------- + x : float + x-axis position + y : float + y-axis position + z : float + z-axis position + + Returns + ------- + fields: numpy.ndarray + + See also + -------- + getFields : returns density, potential and acceleration + getFieldsCoefs : get fields for each coefficient set + __call__ : same as getFields() but provides field labels in a tuple )", py::arg("x"), py::arg("y"), py::arg("z")) .def("getFieldsCoefs", &BasisClasses::BiorthBasis::getFieldsCoefs, From 3b422f881550c7069973071eb98a4c4cfc903120 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 21 May 2025 12:03:41 -0400 Subject: [PATCH 039/106] Compiles successfully and pybind11 interface appears as expected in Python; still needs checking against 'getFields' --- expui/BiorthBasis.cc | 109 +++++++++++++++++++++++++++++++++++++++++ pyEXP/BasisWrappers.cc | 34 +++++++++++++ 2 files changed, 143 insertions(+) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index fc728f242..7fe13fadd 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -505,6 +505,115 @@ namespace BasisClasses } } + std::vector + Spherical::sph_eval(double r, double costh, double phi) + { + // Get thread id + int tid = omp_get_thread_num(); + + double fac1, cosm, sinm; + double sinth = -sqrt(fabs(1.0 - costh*costh)); + + fac1 = factorial(0, 0); + + get_dens (dend[tid], r/scale); + get_pot (potd[tid], r/scale); + get_force(dpot[tid], r/scale); + + legendre_R(lmax, costh, legs[tid], dlegs[tid]); + + double den0, pot0, potr; + + if (NO_L0) { + den0 = 0.0; + pot0 = 0.0; + potr = 0.0; + } else { + den0 = fac1 * expcoef.row(0).dot(dend[tid].row(0)); + pot0 = fac1 * expcoef.row(0).dot(potd[tid].row(0)); + potr = fac1 * expcoef.row(0).dot(dpot[tid].row(0)); + } + + double den1 = 0.0; + double pot1 = 0.0; + double pott = 0.0; + double potp = 0.0; + + // L loop + for (int l=1, loffset=1; l<=lmax; loffset+=(2*l+1), l++) { + + // Check for even l + if (EVEN_L and l%2) continue; + + // No l=1 + if (NO_L1 and l==1) continue; + + // M loop + for (int m=0, moffset=0; m<=l; m++) { + + if (M0_only and m) continue; + if (EVEN_M and m%2) continue; + + fac1 = factorial(l, m); + if (m==0) { + double sumR=0.0, sumP=0.0, sumD=0.0; + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + sumR += expcoef(loffset+moffset, n) * dend[tid](l, n); + sumP += expcoef(loffset+moffset, n) * potd[tid](l, n); + sumD += expcoef(loffset+moffset, n) * dpot[tid](l, n); + } + + den1 += fac1*legs[tid] (l, m) * sumR; + pot1 += fac1*legs[tid] (l, m) * sumP; + potr += fac1*legs[tid] (l, m) * sumD; + pott += fac1*dlegs[tid](l, m) * sumP; + + moffset++; + } + else { + cosm = cos(phi*m); + sinm = sin(phi*m); + + double sumR0=0.0, sumP0=0.0, sumD0=0.0; + double sumR1=0.0, sumP1=0.0, sumD1=0.0; + for (int n=std::max(0, N1); n<=std::min(nmax-1, N2); n++) { + sumR0 += expcoef(loffset+moffset+0, n) * dend[tid](l, n); + sumP0 += expcoef(loffset+moffset+0, n) * potd[tid](l, n); + sumD0 += expcoef(loffset+moffset+0, n) * dpot[tid](l, n); + sumR1 += expcoef(loffset+moffset+1, n) * dend[tid](l, n); + sumP1 += expcoef(loffset+moffset+1, n) * potd[tid](l, n); + sumD1 += expcoef(loffset+moffset+1, n) * dpot[tid](l, n); + } + + den1 += fac1 * legs[tid] (l, m) * ( sumR0*cosm + sumR1*sinm ); + pot1 += fac1 * legs[tid] (l, m) * ( sumP0*cosm + sumP1*sinm ); + potr += fac1 * legs[tid] (l, m) * ( sumD0*cosm + sumD1*sinm ); + pott += fac1 * dlegs[tid](l, m) * ( sumP0*cosm + sumP1*sinm ); + potp += fac1 * legs[tid] (l, m) * (-sumP0*sinm + sumP1*cosm ) * m; + + moffset +=2; + } + } + } + + double densfac = 1.0/(scale*scale*scale) * 0.25/M_PI; + double potlfac = 1.0/scale; + + return + {den0 * densfac, // 0 + den1 * densfac, // 1 + (den0 + den1) * densfac, // 2 + pot0 * potlfac, // 3 + pot1 * potlfac, // 4 + (pot0 + pot1) * potlfac, // 5 + potr * (-potlfac)/scale, // 6 + pott * (-potlfac), // 7 + potp * (-potlfac)}; // 8 + // ^ + // | + // Return force not potential gradient + } + std::vector Spherical::getAccel(double x, double y, double z) { diff --git a/pyEXP/BasisWrappers.cc b/pyEXP/BasisWrappers.cc index ced28b0d1..58042c8fb 100644 --- a/pyEXP/BasisWrappers.cc +++ b/pyEXP/BasisWrappers.cc @@ -426,6 +426,12 @@ void BasisFactoryClasses(py::module &m) PYBIND11_OVERRIDE_PURE(void, BiorthBasis, set_coefs, coefs); } + std::vector getAccel(double x, double y, double z) override + { + PYBIND11_OVERRIDE_PURE(std::vector, BiorthBasis, + getAccel, x, y, z); + } + }; class PySpherical : public Spherical @@ -470,6 +476,10 @@ void BasisFactoryClasses(py::module &m) PYBIND11_OVERRIDE(std::vector, Spherical, getFields, x, y, z); } + std::vector getAccel(double x, double y, double z) override { + PYBIND11_OVERRIDE(std::vector, Spherical, getAccel, x, y, z); + } + void accumulate(double x, double y, double z, double mass) override { PYBIND11_OVERRIDE(void, Spherical, accumulate, x, y, z, mass); } @@ -518,6 +528,10 @@ void BasisFactoryClasses(py::module &m) PYBIND11_OVERRIDE(std::vector, Cylindrical, getFields, x, y, z); } + std::vector getAccel(double x, double y, double z) override { + PYBIND11_OVERRIDE(std::vector, Cylindrical, getAccel, x, y, z); + } + void accumulate(double x, double y, double z, double mass) override { PYBIND11_OVERRIDE(void, Cylindrical, accumulate, x, y, z, mass); } @@ -580,6 +594,11 @@ void BasisFactoryClasses(py::module &m) PYBIND11_OVERRIDE(std::vector, FlatDisk, getFields, x, y, z); } + std::vector getAccel(double x, double y, double z) override + { + PYBIND11_OVERRIDE(std::vector, FlatDisk, getAccel, x, y, z); + } + void accumulate(double x, double y, double z, double mass) override { PYBIND11_OVERRIDE(void, FlatDisk, accumulate, x, y, z, mass); @@ -645,6 +664,11 @@ void BasisFactoryClasses(py::module &m) PYBIND11_OVERRIDE(std::vector, CBDisk, getFields, x, y, z); } + std::vector getAccel(double x, double y, double z) override + { + PYBIND11_OVERRIDE(std::vector, CBDisk, getAccel, x, y, z); + } + void accumulate(double x, double y, double z, double mass) override { PYBIND11_OVERRIDE(void, CBDisk, accumulate, x, y, z, mass); @@ -713,6 +737,11 @@ void BasisFactoryClasses(py::module &m) PYBIND11_OVERRIDE(std::vector, Slab, getFields, x, y, z); } + std::vector getAccel(double x, double y, double z) override + { + PYBIND11_OVERRIDE(std::vector, Slab, getAccel, x, y, z); + } + void accumulate(double x, double y, double z, double mass) override { PYBIND11_OVERRIDE(void, Slab, accumulate, x, y, z, mass); @@ -781,6 +810,11 @@ void BasisFactoryClasses(py::module &m) PYBIND11_OVERRIDE(std::vector, Cube, getFields, x, y, z); } + std::vector getAccel(double x, double y, double z) override + { + PYBIND11_OVERRIDE(std::vector, Cube, getAccel, x, y, z); + } + void accumulate(double x, double y, double z, double mass) override { PYBIND11_OVERRIDE(void, Cube, accumulate, x, y, z, mass); From c6a69caafc191f325853758564241c7b2670aa8e Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 21 May 2025 12:21:08 -0400 Subject: [PATCH 040/106] Typo fix in exception message --- expui/BiorthBasis.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 7fe13fadd..e3b03b1ea 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -4209,7 +4209,8 @@ namespace BasisClasses auto it2 = std::lower_bound(times.begin(), times.end(), t); auto it1 = it2; - if (it2 == times.end()) throw std::runtime_error("Basis::AllTimeAccel::evalcoefs: time t=" + std::to_string(t) + " out of bounds"); + if (it2 == times.end()) + throw std::runtime_error("Basis::SingleTimeAccel::evalcoefs: time t=" + std::to_string(t) + " out of bounds"); else if (it2 == times.begin()) it2++; else it1--; From 13482640e70411315765e934c903259bab9890cf Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 21 May 2025 16:30:26 -0400 Subject: [PATCH 041/106] Some code clean-up; fixed a sign typo for Spherical --- expui/BiorthBasis.H | 3 +-- expui/BiorthBasis.cc | 19 ++++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/expui/BiorthBasis.H b/expui/BiorthBasis.H index 0fd25c090..982002065 100644 --- a/expui/BiorthBasis.H +++ b/expui/BiorthBasis.H @@ -166,8 +166,7 @@ namespace BasisClasses //! Evaluate acceleration in Cartesian coordinates in centered //! coordinate system - virtual std::vector - getAccel(double x, double y, double z) = 0; + virtual std::vector getAccel(double x, double y, double z) = 0; }; /** diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index e3b03b1ea..82030700c 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -511,10 +511,7 @@ namespace BasisClasses // Get thread id int tid = omp_get_thread_num(); - double fac1, cosm, sinm; - double sinth = -sqrt(fabs(1.0 - costh*costh)); - - fac1 = factorial(0, 0); + double fac1 = factorial(0, 0); get_dens (dend[tid], r/scale); get_pot (potd[tid], r/scale); @@ -571,8 +568,8 @@ namespace BasisClasses moffset++; } else { - cosm = cos(phi*m); - sinm = sin(phi*m); + double cosm = cos(phi*m); + double sinm = sin(phi*m); double sumR0=0.0, sumP0=0.0, sumD0=0.0; double sumR1=0.0, sumP1=0.0, sumD1=0.0; @@ -617,18 +614,18 @@ namespace BasisClasses std::vector Spherical::getAccel(double x, double y, double z) { + // Get polar coordinates double R = sqrt(x*x + y*y); double r = sqrt(R*R + z*z); double costh = z/r; + double sinth = R/r; double phi = atan2(y, x); // Get thread id int tid = omp_get_thread_num(); - double fac1, cosm, sinm; - double sinth = -sqrt(fabs(1.0 - costh*costh)); - fac1 = factorial(0, 0); + double fac1 = factorial(0, 0); get_pot (potd[tid], r/scale); get_force(dpot[tid], r/scale); @@ -681,8 +678,8 @@ namespace BasisClasses moffset++; } else { - cosm = cos(phi*m); - sinm = sin(phi*m); + double cosm = cos(phi*m); + double sinm = sin(phi*m); double sumR0=0.0, sumP0=0.0, sumD0=0.0; double sumR1=0.0, sumP1=0.0, sumD1=0.0; From 7181b7273b6df925a0b4ea4ae353c180754925fd Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 21 May 2025 22:15:08 -0400 Subject: [PATCH 042/106] Some additional code clean-up; fixed an index issue and coordinate transform for FlatDisk and CBDisk --- expui/BiorthBasis.cc | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 82030700c..6c55e4a28 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -2099,7 +2099,7 @@ namespace BasisClasses return {den0, den1, den0+den1, pot0, pot1, pot0+pot1, rpot, zpot, ppot}; } - std::vectorFlatDisk::getAccel(double R, double z, double phi) + std::vector FlatDisk::getAccel(double x, double y, double z) { // Get thread id int tid = omp_get_thread_num(); @@ -2108,6 +2108,11 @@ namespace BasisClasses constexpr double norm0 = 0.5*M_2_SQRTPI/M_SQRT2; constexpr double norm1 = 0.5*M_2_SQRTPI; + // Compute polar coordinates + double R = std::sqrt(x*x + y*y); + double phi = std::atan2(y, x); + + double rpot=0, zpot=0, ppot=0; // Off grid evaluation @@ -2179,7 +2184,10 @@ namespace BasisClasses zpot *= -1.0; ppot *= -1.0; - return {rpot, zpot, ppot}; + double potx = rpot*x/R - ppot*y/R; + double poty = rpot*y/R + ppot*x/R; + + return {potx, poty, zpot}; } @@ -2209,8 +2217,8 @@ namespace BasisClasses auto v = cyl_eval(R, z, phi); - double potx = v[4]*x/R - v[8]*y/R; - double poty = v[4]*y/R + v[8]*x/R; + double potx = v[6]*x/R - v[8]*y/R; + double poty = v[6]*y/R + v[8]*x/R; return {v[0], v[1], v[2], v[3], v[4], v[5], potx, poty, v[7]}; } @@ -2785,7 +2793,7 @@ namespace BasisClasses } } - std::vectorCBDisk::cyl_eval(double R, double z, double phi) + std::vector CBDisk::cyl_eval(double R, double z, double phi) { // Get thread id int tid = omp_get_thread_num(); @@ -2863,7 +2871,7 @@ namespace BasisClasses } - std::vectorCBDisk::getAccel(double x, double y, double z) + std::vector CBDisk::getAccel(double x, double y, double z) { // Get thread id int tid = omp_get_thread_num(); @@ -2924,7 +2932,11 @@ namespace BasisClasses rpot *= -1.0; ppot *= -1.0; - return {rpot, zpot, ppot}; + + double potx = rpot*x/R - ppot*y/R; + double poty = rpot*y/R + ppot*x/R; + + return {potx, poty, zpot}; } @@ -2954,8 +2966,8 @@ namespace BasisClasses auto v = cyl_eval(R, z, phi); - double potx = v[4]*x/R - v[8]*y/R; - double poty = v[4]*y/R + v[8]*x/R; + double potx = v[6]*x/R - v[8]*y/R; + double poty = v[6]*y/R + v[8]*x/R; return {v[0], v[1], v[2], v[3], v[4], v[5], potx, poty, v[7]}; } @@ -3340,7 +3352,7 @@ namespace BasisClasses // Working values // - std::complex facx, facy, fac, facf, facd; + std::complex facx, facy, fac, facf; // Return values // @@ -3356,7 +3368,7 @@ namespace BasisClasses std::complex startx = exp(-static_cast(nmaxx)*kfac*x); std::complex starty = exp(-static_cast(nmaxy)*kfac*y); - Eigen::VectorXd vpot(nmaxz), vfrc(nmaxz), vden(nmaxz); + Eigen::VectorXd vpot(nmaxz), vfrc(nmaxz); for (facx=startx, ix=0; ix=iiy) { + ortho->get_pot (vpot, z, iix, iiy); ortho->get_force(vfrc, z, iix, iiy); } else { + ortho->get_pot (vpot, z, iiy, iix); ortho->get_force(vfrc, z, iiy, iix); } @@ -3390,7 +3404,6 @@ namespace BasisClasses fac = facx*facy*vpot[iz]*expcoef(ix, iy, iz); facf = facx*facy*vfrc[iz]*expcoef(ix, iy, iz); - facd = facx*facy*vden[iz]*expcoef(ix, iy, iz); // Limit to minimum wave number // From f1cd74ff0b0b8da94be6f46eae0dbe1dd0748ca7 Mon Sep 17 00:00:00 2001 From: michael-petersen Date: Thu, 22 May 2025 15:32:01 -0600 Subject: [PATCH 043/106] bring cylcache in line with sech2 --- utils/ICs/cylcache.cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/utils/ICs/cylcache.cc b/utils/ICs/cylcache.cc index 1a1249e7f..e23a3cc65 100644 --- a/utils/ICs/cylcache.cc +++ b/utils/ICs/cylcache.cc @@ -184,6 +184,7 @@ double ASHIFT; double HSCALE; double HERNA; double Mfac; +bool sech2; double RTRUNC = 1.0; double RWIDTH = 0.0; double ARATIO = 1.0; @@ -224,8 +225,8 @@ double DiskDens(double R, double z, double phi) { double a1 = ASCALE; double a2 = ASCALE*ARATIO; - double h1 = HSCALE; - double h2 = HSCALE*HRATIO; + double h1 = sech2 ? 0.5*HSCALE : HSCALE; + double h2 = h1*HRATIO; double w1 = 1.0/(1.0+DWEIGHT); double w2 = DWEIGHT/(1.0+DWEIGHT); @@ -239,13 +240,14 @@ double DiskDens(double R, double z, double phi) break; case DiskType::diskbulge: { - double f = cosh(z/HSCALE); + double h = sech2 ? 0.5*HSCALE : HSCALE; + double f = cosh(z/h); double rr = pow(pow(R, 2) + pow(z,2), 0.5); double w1 = Mfac; double w2 = (1-Mfac); double as = HERNA; - ans = w1*exp(-R/ASCALE)/(4.0*M_PI*ASCALE*ASCALE*HSCALE*f*f) + + ans = w1*exp(-R/ASCALE)/(4.0*M_PI*ASCALE*ASCALE*h*f*f) + w2*pow(as, 4)/(2.0*M_PI*rr)*pow(rr+as,-3.0) ; } break; @@ -258,8 +260,9 @@ double DiskDens(double R, double z, double phi) case DiskType::exponential: default: { - double f = cosh(z/HSCALE); - ans = exp(-R/ASCALE)/(4.0*M_PI*ASCALE*ASCALE*HSCALE*f*f); + double h = sech2 ? 0.5*HSCALE : HSCALE; + double f = cosh(z/h); + ans = exp(-R/ASCALE)/(4.0*M_PI*ASCALE*ASCALE*h*f*f); } break; } @@ -372,7 +375,9 @@ main(int ac, char **av) ("expcond", "Use analytic target density rather than particle distribution", cxxopts::value(expcond)->default_value("true")) ("LOGR", "Logarithmic scaling for model table in EmpCylSL", - cxxopts::value(LOGR)->default_value("true")) + cxxopts::value(expcond)->default_value("true")) + ("sech2", "if true, use hcyl as sech^2(z/(2*hcyl))", + cxxopts::value(sech2)->default_value("false")) ("RCYLMIN", "Minimum disk radius for EmpCylSL", cxxopts::value(RCYLMIN)->default_value("0.001")) ("RCYLMAX", "Maximum disk radius for EmpCylSL", @@ -580,6 +585,7 @@ main(int ac, char **av) << " rmax=" << EmpCylSL::RMAX << " a=" << ASCALE << " h=" << HSCALE + << " sech2=" << sech2 << " as=" << HERNA << " Mfac=" << Mfac << " nmax2=" << NMAXFID @@ -597,7 +603,8 @@ main(int ac, char **av) // The scale in EmpCylSL is assumed to be 1 so we compute the // height relative to the length // - double H = HSCALE/ASCALE; + //double H = HSCALE/ASCALE; + double H = sech2 ? 0.5*HSCALE/ASCALE : HSCALE/ASCALE; // The model instance (you can add others in DiskModels.H). // It's MN or Exponential if not MN. From 2fc12fe999f8edfb4eff8ecee0f40bdf05ebbb51 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 25 May 2025 12:18:10 -0400 Subject: [PATCH 044/106] Cosmetic changes to log output --- expui/BiorthBasis.H | 20 ++++++++++++++------ exputil/orthoTest.cc | 2 +- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/expui/BiorthBasis.H b/expui/BiorthBasis.H index 5e437e237..1c213061c 100644 --- a/expui/BiorthBasis.H +++ b/expui/BiorthBasis.H @@ -303,7 +303,8 @@ namespace BasisClasses { auto [ret, worst, lworst] = orthoCompute(orthoCheck(knots)); // For the CTest log - std::cout << "---- Spherical::orthoTest: worst=" << worst << std::endl; + if (myid==0) + std::cout << "---- Spherical::orthoTest: worst=" << worst << std::endl; return ret; } @@ -538,7 +539,8 @@ namespace BasisClasses { auto [ret, worst, lworst] = orthoCompute(orthoCheck()); // For the CTest log - std::cout << "---- FlatDisk::orthoTest: worst=" << worst << std::endl; + if (myid==0) + std::cout << "---- FlatDisk::orthoTest: worst=" << worst << std::endl; return ret; } @@ -689,7 +691,8 @@ namespace BasisClasses { auto [ret, worst, lworst] = orthoCompute(orthoCheck()); // For the CTest log - std::cout << "CBDisk::orthoTest: worst=" << worst << std::endl; + if (myid==0) + std::cout << "CBDisk::orthoTest: worst=" << worst << std::endl; return ret; } @@ -842,7 +845,8 @@ namespace BasisClasses { auto [ret, worst, lworst] = orthoCompute(sl->orthoCheck()); // For the CTest log - std::cout << "---- Cylindrical::orthoTest: worst=" << worst << std::endl; + if (myid==0) + std::cout << "---- Cylindrical::orthoTest: worst=" << worst << std::endl; return ret; } }; @@ -985,8 +989,10 @@ namespace BasisClasses } } } - + + if (myid==0) std::cout << "---- Slab::orthoTest: worst=" << worst << std::endl; + if (worst > __EXP__::orthoTol) return false; return true; } @@ -1108,7 +1114,9 @@ namespace BasisClasses } } - std::cout << "---- Cube::orthoTest: worst=" << worst << std::endl; + if (myid==0) + std::cout << "---- Cube::orthoTest: worst=" << worst << std::endl; + if (worst > __EXP__::orthoTol) return false; return true; } diff --git a/exputil/orthoTest.cc b/exputil/orthoTest.cc index 99fcb4f2e..9cfb8c82e 100644 --- a/exputil/orthoTest.cc +++ b/exputil/orthoTest.cc @@ -81,6 +81,6 @@ void orthoTest(const std::vector& tests, } else { // Success message if (myid==0) - std::cout << classname + ": biorthogonal check passed" << std::endl; + std::cout << "---- " << classname + ": biorthogonal check passed" << std::endl; } } From b47b0c3b69cf97449910e2d88cd06e1d6385de3d Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 28 May 2025 16:30:13 -0400 Subject: [PATCH 045/106] Preliminarly, incomplete check-in of HDF5 phase-space implementation --- src/CMakeLists.txt | 28 ++-- src/Component.H | 5 + src/Component.cc | 72 ++++++++++ src/OutHDF5.H | 49 +++++++ src/OutHDF5.cc | 294 +++++++++++++++++++++++++++++++++++++++++ src/OutputContainer.cc | 5 + 6 files changed, 439 insertions(+), 14 deletions(-) create mode 100644 src/OutHDF5.H create mode 100644 src/OutHDF5.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3f712eafe..976663b3a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,17 +1,17 @@ -set(exp_SOURCES Basis.cc Bessel.cc Component.cc - Cube.cc Cylinder.cc ExternalCollection.cc CBDisk.cc - ExternalForce.cc Orient.cc PotAccel.cc ScatterMFP.cc - PeriodicBC.cc SphericalBasis.cc AxisymmetricBasis.cc Sphere.cc - TwoDCoefs.cc TwoCenter.cc EJcom.cc global.cc begin.cc ddplgndr.cc - Direct.cc Shells.cc NoForce.cc end.cc OutputContainer.cc OutPS.cc - OutPSQ.cc OutPSN.cc OutPSP.cc OutPSR.cc OutCHKPT.cc OutCHKPTQ.cc - Output.cc externalShock.cc CylEXP.cc generateRelaxation.cc - HaloBulge.cc incpos.cc incvel.cc ComponentContainer.cc OutAscii.cc - OutMulti.cc OutRelaxation.cc OrbTrace.cc OutDiag.cc OutLog.cc - OutVel.cc OutCoef.cc multistep.cc parse.cc SlabSL.cc step.cc - tidalField.cc ultra.cc ultrasphere.cc MPL.cc OutFrac.cc OutCalbr.cc - ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc - CenterFile.cc PolarBasis.cc FlatDisk.cc signals.cc) +set(exp_SOURCES Basis.cc Bessel.cc Component.cc Cube.cc Cylinder.cc + ExternalCollection.cc CBDisk.cc ExternalForce.cc Orient.cc + PotAccel.cc ScatterMFP.cc PeriodicBC.cc SphericalBasis.cc + AxisymmetricBasis.cc Sphere.cc TwoDCoefs.cc TwoCenter.cc EJcom.cc + global.cc begin.cc ddplgndr.cc Direct.cc Shells.cc NoForce.cc end.cc + OutputContainer.cc OutPS.cc OutPSQ.cc OutPSN.cc OutPSP.cc OutPSR.cc + OutCHKPT.cc OutCHKPTQ.cc Output.cc externalShock.cc CylEXP.cc + generateRelaxation.cc HaloBulge.cc incpos.cc incvel.cc + ComponentContainer.cc OutAscii.cc OutHDF5.cc OutMulti.cc + OutRelaxation.cc OrbTrace.cc OutDiag.cc OutLog.cc OutVel.cc + OutCoef.cc multistep.cc parse.cc SlabSL.cc step.cc tidalField.cc + ultra.cc ultrasphere.cc MPL.cc OutFrac.cc OutCalbr.cc + ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc CenterFile.cc + PolarBasis.cc FlatDisk.cc signals.cc) if (ENABLE_CUDA) list(APPEND exp_SOURCES cudaPolarBasis.cu cudaSphericalBasis.cu diff --git a/src/Component.H b/src/Component.H index 82eae2ce2..d141e004e 100644 --- a/src/Component.H +++ b/src/Component.H @@ -349,6 +349,7 @@ protected: static const std::set valid_keys_force; //@} + public: //! Describe the phase space coordinates @@ -551,6 +552,10 @@ public: //! Write binary component phase-space structure void write_binary(ostream *out, bool real4 = false); + //! Write HDF5 phase-space structure for real or float + template + void write_HDF5(HighFive::Group& group, bool masses, bool IDs); + //! Write header for per-node writes void write_binary_header(ostream* out, bool real4, std::string prefix, int nth=1); diff --git a/src/Component.cc b/src/Component.cc index b52f30c30..c694f6e9a 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2247,6 +2247,78 @@ void Component::write_binary(ostream* out, bool real4) } +//! Write HDF5 phase-space structure +template +void Component::write_HDF5_type(HighFive::Group& group, + bool masses = true, + bool IDs = true) +{ + Eigen::Matrix pos(nbodies, 3); + Eigen::Matrix vel(nbodies, 3); + + std::vector mas; + std::vector ids; + std::vector> iattrib; + std::vector> rattrib; + + if (masses) mas.resize(nbodies); + if (IDs) ids.resize(nbodies); + if (niattrib) { + iattrib.resize(niattrib); + for (auto & v : iattrib) v.resize(nbodies); + } + if (ndattrib) { + rattrib.resize(ndattrib); + for (auto & v : rattrib) v.resize(nbodies); + } + + unsigned int ctr = 0; + + // First bunch of particles + int number = -1; + PartPtr *p = get_particles(&number); + + // Keep going... + while (p) { + for (int k=0; kmass(); + if (IDs) ids[cnt] = p->index; + for (int j=0; j<3; j++) pos(cnt, j) = p[k]->pos[j]; + for (int j=0; j<3; j++) vel(cnt, j) = p[k]->vel[j]; + for (int j=0; jiattrib[j]; + for (int j=0; jdattrib[j]; + // Increment array position + cnt++; + } + // Next bunch of particles + p = get_particles(&number); + } + + // Add all datasets + if (masses) { + HighFive::DataSet ds = group.createDataSet("Masses", mas); + } + + if (IDs) { + HighFive::DataSet ds = group.createDataSet("ParticleIDs", ids); + } + + HighFive::DataSet dsPos = group.createDataSet("Positions", pos); + HighFive::DataSet dsVel = group.createDataSet("Velocities", vel); + + if (int n=0; n + +/** Write phase-space dumps at regular intervals from each node in + component pieces. This writer is indended to be Gadget-4 HDF5 + compliant. + + @param filename is the name of the output file + @param nint is the number of steps between dumps + @param nbeg is suffix of the first phase space %dump + @param timer set to true turns on wall-clock timer for PS output + @param threads number of threads for binary writes + +*/ +class OutHDF5 : public Output +{ + +private: + + std::string filename; + bool real4=true, real8=false, timer; + int nbeg, threads; + void initialize(void); + + //! Valid keys for YAML configurations + static const std::set valid_keys; + +public: + + //! Constructor + OutHDF5(const YAML::Node & conf); + + //! Provided by derived class to generate some output + /*! + \param nstep is the current time step used to decide whether or not + to %dump + \param mstep is the current multistep level to decide whether or not to dump multisteps + \param last should be true on final step to force phase space %dump + indepentently of whether or not the frequency criterion is met + \param timer set to true turns on wall-clock timer for PS output + \param threads is the thread count for binary writes + */ + void Run(int nstep, int mstep, bool last); + +}; + +#endif diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc new file mode 100644 index 000000000..2db366ae1 --- /dev/null +++ b/src/OutHDF5.cc @@ -0,0 +1,294 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "expand.H" +#include + +#include + +const std::set +OutHDF5::valid_keys = { + "filename", + "nint", + "nintsub", + "nbeg", + "real4", + "real8", + "timer", + "threads" +}; + +OutHDF5::OutHDF5(const YAML::Node& conf) : Output(conf) +{ + initialize(); +} + +void OutHDF5::initialize() +{ + // Remove matched keys + // + for (auto v : valid_keys) current_keys.erase(v); + + // Assign values from YAML + // + try { + // Get file name + if (Output::conf["filename"]) + filename = Output::conf["filename"].as(); + else{ + filename.erase(); + filename = "H5_" + runtag; + } + + if (Output::conf["nint"]) + nint = Output::conf["nint"].as(); + else + nint = 100; + + if (Output::conf["nintsub"]) { +#ifdef ALLOW_NINTSUB + nintsub = Output::conf["nintsub"].as(); + if (nintsub <= 0) nintsub = 1; +#else + nintsub_warning("OutHDF5"); + nintsub = std::numeric_limits::max(); +#endif + } else + nintsub = std::numeric_limits::max(); + + if (Output::conf["nbeg"]) + nbeg = Output::conf["nbeg"].as(); + else + nbeg = 0; + + if (Output::conf["real4"]) { + real4 = Output::conf["real4"].as(); + real8 = not real4; + } else { + real4 = true; + real8 = false; + } + + if (Output::conf["real8"]) { + real8 = Output::conf["real8"].as(); + real4 = not real8; + } + + if (Output::conf["timer"]) + timer = Output::conf["timer"].as(); + else + timer = false; + + if (Output::conf["threads"]) + threads = Output::conf["threads"].as(); + else + threads = 0; + } + catch (YAML::Exception & error) { + if (myid==0) std::cout << "Error parsing parameters in OutHDF5: " + << error.what() << std::endl + << std::string(60, '-') << std::endl + << "Config node" << std::endl + << std::string(60, '-') << std::endl + << conf << std::endl + << std::string(60, '-') << std::endl; + throw std::runtime_error("OutHDF5::initialize: error parsing YAML"); + } + + // Determine last file + // + if (restart && nbeg==0) { + + // Only root node looks for files + // + if (myid==0) { + + for (nbeg=0; nbeg<100000; nbeg++) { + // Output name + // + ostringstream fname; + fname << outdir + << filename << "_" << setw(5) << setfill('0') << nbeg + << ".1"; + + // See if we can open file + // + ifstream in(fname.str().c_str()); + + if (!in) { + cout << "OutHDF5: will begin with nbeg=" << nbeg << endl; + break; + } + } + } + + // Communicate starting file to all nodes + // + MPI_Bcast(&nbeg, 1, MPI_INT, 0, MPI_COMM_WORLD); + } +} + + +void OutHDF5::Run(int n, int mstep, bool last) +{ + if (!dump_signal and !last) { + if (n % nint) return; + if (restart && n==0) return; + if (multistep>1 && mstep % nintsub !=0) return; + } + + std::chrono::high_resolution_clock::time_point beg, end; + if (timer) beg = std::chrono::high_resolution_clock::now(); + + std::ofstream out; + std::ostringstream fname; + + // Output name prefix + fname << filename << "." << setw(5) << setfill('0') << nbeg++ + << "." << myid+1; + + // Master file name + std::string path = outdir + fname.str(); + + int nOK = 0; + + HighFive::File file; + + // Begin HDF5 file writing + // + try { + + // Silence the HDF5 error stack + // + HighFive::SilenceHDF5 quiet; + + // Try opening the file as HDF5 + // + file = HighFive::File(path, + HighFive::File::ReadWrite | + HighFive::File::Create); + try { + + // Create a new group for Header + // + HighFive::Group header = file.createGroup("Header"); + + // Create a new group for Config + // + HighFive::Group config = file.createGroup("Config"); + + // Create a new group for Parameters + // + HighFive::Group params = file.createGroup("Parameters"); + + } catch (HighFive::Exception& err) { + std::string msg("Coefs::factory: error reading HDF5 file, "); + throw std::runtime_error(msg + err.what()); + } + + } catch (HighFive::Exception& err) { + std::cerr << "OutHDF5 [" << myid << "]: can't open file <" << path + << "> . . . quitting" << std::endl; + nOK = 1; + } + + MPI_Allreduce(0, &nOK, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + // Exit on file open failure + // + if (nOK) { + MPI_Finalize(); + exit(33); + } + + + int count = 0; + for (auto c : comp->components) { + +#ifdef HAVE_LIBCUDA + if (use_cuda) { + if (c->force->cudaAware() and not comp->fetched[c]) { + comp->fetched[c] = true; + c->CudaToParticles(); + } + } +#endif + + std::ostringstream sout; + sout << "PartType" << count++; + + file.createGroup(sout.str()); + + + if (myid==0) { + c->write_binary_header(&out, real4, cname.str()); + } + + cname << "-" << myid; + + // Open particle file and write + std::string blobfile = outdir + cname.str(); + std::ofstream pout(blobfile); + + if (pout.fail()) { + std::cerr << "[" << myid << "] OutHDF5: can't open file <" << cname.str() + << "> . . . quitting" << std::endl; + nOK = 1; + } else { + if (threads) + c->write_binary_particles(&pout, threads, real4); + else + c->write_binary_particles(&pout, real4); + if (pout.fail()) { + std::cout << "OutHDF5: error writing binary particles to <" + << blobfile << std::endl; + } + } + + + // Check for errors in all file opening + int sumOK; + MPI_Allreduce(&nOK, &sumOK, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + if (sumOK) { + MPI_Finalize(); + exit(35); + } + } + + if (myid==0) { + if (out.fail()) { + std::cout << "OutHDF5: error writing component to master <" << master + << std::endl; + } + + try { + out.close(); + } + catch (const ofstream::failure& e) { + std::cout << "OutHDF5: exception closing file <" << master + << ": " << e.what() << std::endl; + } + } + + chktimer.mark(); + + dump_signal = 0; + + if (timer) { + end = std::chrono::high_resolution_clock::now(); + std::chrono::duration intvl = end - beg; + if (myid==0) + std::cout << "OutHDF5 [T=" << tnow << "] timing=" << intvl.count() + << std::endl; + } +} + diff --git a/src/OutputContainer.cc b/src/OutputContainer.cc index 8184d407f..189f7ee15 100644 --- a/src/OutputContainer.cc +++ b/src/OutputContainer.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -71,6 +72,10 @@ void OutputContainer::initialize(void) out.push_back(new OutPSQ (node)); } + else if ( !name.compare("outhdf5") ) { + out.push_back(new OutHDF5 (node)); + } + else if ( !name.compare("outpsr") ) { out.push_back(new OutPSR (node)); } From c7a1a43b21ac147e50c4ec9f6a57212c783266b3 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 28 May 2025 17:35:43 -0400 Subject: [PATCH 046/106] Add some metadata writing; make it compile --- src/Component.H | 7 +++- src/Component.cc | 34 +++++++++------ src/OutHDF5.cc | 106 ++++++++++++++++++++--------------------------- 3 files changed, 73 insertions(+), 74 deletions(-) diff --git a/src/Component.H b/src/Component.H index d141e004e..8f7adfab7 100644 --- a/src/Component.H +++ b/src/Component.H @@ -6,6 +6,11 @@ #include +#include +#include +#include +#include + #include #include #include @@ -553,7 +558,7 @@ public: void write_binary(ostream *out, bool real4 = false); //! Write HDF5 phase-space structure for real or float - template + template void write_HDF5(HighFive::Group& group, bool masses, bool IDs); //! Write header for per-node writes diff --git a/src/Component.cc b/src/Component.cc index c694f6e9a..39a4ccc99 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2248,13 +2248,11 @@ void Component::write_binary(ostream* out, bool real4) } //! Write HDF5 phase-space structure -template -void Component::write_HDF5_type(HighFive::Group& group, - bool masses = true, - bool IDs = true) +template +void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) { - Eigen::Matrix pos(nbodies, 3); - Eigen::Matrix vel(nbodies, 3); + Eigen::Matrix pos(nbodies, 3); + Eigen::Matrix vel(nbodies, 3); std::vector mas; std::vector ids; @@ -2281,14 +2279,15 @@ void Component::write_HDF5_type(HighFive::Group& group, // Keep going... while (p) { for (int k=0; kmass(); - if (IDs) ids[cnt] = p->index; - for (int j=0; j<3; j++) pos(cnt, j) = p[k]->pos[j]; - for (int j=0; j<3; j++) vel(cnt, j) = p[k]->vel[j]; - for (int j=0; jiattrib[j]; - for (int j=0; jdattrib[j]; + auto &P = *p; + if (masses) mas[ctr] = P->mass; + if (IDs) ids[ctr] = P->indx; + for (int j=0; j<3; j++) pos(ctr, j) = p[k]->pos[j]; + for (int j=0; j<3; j++) vel(ctr, j) = p[k]->vel[j]; + for (int j=0; jiattrib[j]; + for (int j=0; jdattrib[j]; // Increment array position - cnt++; + ctr++; } // Next bunch of particles p = get_particles(&number); @@ -2319,6 +2318,15 @@ void Component::write_HDF5_type(HighFive::Group& group, } } +// Explicit instantiations for float and double +template +void Component::write_HDF5 +(HighFive::Group& group, bool masses, bool IDs); + +template +void Component::write_HDF5 +(HighFive::Group& group, bool masses, bool IDs); + void Component::write_binary_header(ostream* out, bool real4, const std::string prefix, int nth) { ComponentHeader header; diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 2db366ae1..e40f9b18f 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -160,7 +160,7 @@ void OutHDF5::Run(int n, int mstep, bool last) int nOK = 0; - HighFive::File file; + std::unique_ptr file; // Begin HDF5 file writing // @@ -172,22 +172,54 @@ void OutHDF5::Run(int n, int mstep, bool last) // Try opening the file as HDF5 // - file = HighFive::File(path, - HighFive::File::ReadWrite | - HighFive::File::Create); + file = std::make_unique(path, + HighFive::File::ReadWrite | + HighFive::File::Create); try { - // Create a new group for Header + // Create a new group for Config // - HighFive::Group header = file.createGroup("Header"); + HighFive::Group config = file->createGroup("Config"); - // Create a new group for Config + int ncomp = comp->components.size(); + config.createAttribute("NTYPES", HighFive::DataSpace::From(ncomp)).write(ncomp); + std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); + config.createAttribute("Git_commit", HighFive::DataSpace::From(gcommit)).write(gcommit); + config.createAttribute("Git_branch", HighFive::DataSpace::From(gbranch)).write(gbranch); + config.createAttribute("Compile_date", HighFive::DataSpace::From(gdate)).write(gdate); + + // Create a new group for Header // - HighFive::Group config = file.createGroup("Config"); + HighFive::Group header = file->createGroup("Header"); + int dp = 1; + header.createAttribute("Flag_DoublePrecision", HighFive::DataSpace::From(dp)).write(dp); + double hubble = 1, zero = 0; + header.createAttribute("HubbleParam", HighFive::DataSpace::From(hubble)).write(hubble); + header.createAttribute("Omega0", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("OmegaBaryon", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("OmegaLambda", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("Redshift", HighFive::DataSpace::From(zero)).write(zero); + std::vector masses(ncomp, 0.0); + header.createAttribute>("MassTable", HighFive::DataSpace::From(masses)).write(masses); + header.createAttribute("NumFilesPerSnapshot", HighFive::DataSpace::From(numprocs)).write(numprocs); + + std::vector nums(ncomp); + { + int n=0; + for (auto c : comp->components) nums[n++] = c->Number(); + } + header.createAttribute>("NumPart_ThisFile", HighFive::DataSpace::From(nums)).write(nums); + { + int n=0; + for (auto c : comp->components) nums[n++] = c->CurTotal(); + } + header.createAttribute>("NumPart_Total", HighFive::DataSpace::From(nums)).write(nums); + + header.createAttribute("Time", HighFive::DataSpace::From(tnow)).write(tnow); // Create a new group for Parameters // - HighFive::Group params = file.createGroup("Parameters"); + HighFive::Group params = file->createGroup("Parameters"); } catch (HighFive::Exception& err) { std::string msg("Coefs::factory: error reading HDF5 file, "); @@ -225,58 +257,12 @@ void OutHDF5::Run(int n, int mstep, bool last) std::ostringstream sout; sout << "PartType" << count++; - file.createGroup(sout.str()); - - - if (myid==0) { - c->write_binary_header(&out, real4, cname.str()); - } - - cname << "-" << myid; - - // Open particle file and write - std::string blobfile = outdir + cname.str(); - std::ofstream pout(blobfile); - - if (pout.fail()) { - std::cerr << "[" << myid << "] OutHDF5: can't open file <" << cname.str() - << "> . . . quitting" << std::endl; - nOK = 1; - } else { - if (threads) - c->write_binary_particles(&pout, threads, real4); - else - c->write_binary_particles(&pout, real4); - if (pout.fail()) { - std::cout << "OutHDF5: error writing binary particles to <" - << blobfile << std::endl; - } - } - - - // Check for errors in all file opening - int sumOK; - MPI_Allreduce(&nOK, &sumOK, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); - - if (sumOK) { - MPI_Finalize(); - exit(35); - } - } + auto pgroup = file->createGroup(sout.str()); - if (myid==0) { - if (out.fail()) { - std::cout << "OutHDF5: error writing component to master <" << master - << std::endl; - } - - try { - out.close(); - } - catch (const ofstream::failure& e) { - std::cout << "OutHDF5: exception closing file <" << master - << ": " << e.what() << std::endl; - } + if (real4) + c->write_HDF5(pgroup, true, true); + else + c->write_HDF5(pgroup, true, true); } chktimer.mark(); From 05c788cea68ee9afc39fc98be126cb1d85853c9b Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 28 May 2025 17:41:57 -0400 Subject: [PATCH 047/106] Some white space for readability --- src/OutHDF5.cc | 55 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index e40f9b18f..071a388f8 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -182,40 +182,65 @@ void OutHDF5::Run(int n, int mstep, bool last) HighFive::Group config = file->createGroup("Config"); int ncomp = comp->components.size(); - config.createAttribute("NTYPES", HighFive::DataSpace::From(ncomp)).write(ncomp); + config.createAttribute("NTYPES", + HighFive::DataSpace::From(ncomp)).write(ncomp); + std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); - config.createAttribute("Git_commit", HighFive::DataSpace::From(gcommit)).write(gcommit); - config.createAttribute("Git_branch", HighFive::DataSpace::From(gbranch)).write(gbranch); - config.createAttribute("Compile_date", HighFive::DataSpace::From(gdate)).write(gdate); + config.createAttribute("Git_commit", + HighFive::DataSpace::From(gcommit)).write(gcommit); + + config.createAttribute("Git_branch", + HighFive::DataSpace::From(gbranch)).write(gbranch); + + config.createAttribute("Compile_date", + HighFive::DataSpace::From(gdate)).write(gdate); // Create a new group for Header // HighFive::Group header = file->createGroup("Header"); int dp = 1; - header.createAttribute("Flag_DoublePrecision", HighFive::DataSpace::From(dp)).write(dp); + header.createAttribute("Flag_DoublePrecision", + HighFive::DataSpace::From(dp)).write(dp); + double hubble = 1, zero = 0; - header.createAttribute("HubbleParam", HighFive::DataSpace::From(hubble)).write(hubble); - header.createAttribute("Omega0", HighFive::DataSpace::From(zero)).write(zero); - header.createAttribute("OmegaBaryon", HighFive::DataSpace::From(zero)).write(zero); - header.createAttribute("OmegaLambda", HighFive::DataSpace::From(zero)).write(zero); - header.createAttribute("Redshift", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("HubbleParam", + HighFive::DataSpace::From(hubble)).write(hubble); + + header.createAttribute("Omega0", + HighFive::DataSpace::From(zero)).write(zero); + + header.createAttribute("OmegaBaryon", + HighFive::DataSpace::From(zero)).write(zero); + + header.createAttribute("OmegaLambda", + HighFive::DataSpace::From(zero)).write(zero); + + header.createAttribute("Redshift", + HighFive::DataSpace::From(zero)).write(zero); + std::vector masses(ncomp, 0.0); - header.createAttribute>("MassTable", HighFive::DataSpace::From(masses)).write(masses); - header.createAttribute("NumFilesPerSnapshot", HighFive::DataSpace::From(numprocs)).write(numprocs); + header.createAttribute>("MassTable", + HighFive::DataSpace::From(masses)).write(masses); + + header.createAttribute("NumFilesPerSnapshot", + HighFive::DataSpace::From(numprocs)).write(numprocs); std::vector nums(ncomp); { int n=0; for (auto c : comp->components) nums[n++] = c->Number(); } - header.createAttribute>("NumPart_ThisFile", HighFive::DataSpace::From(nums)).write(nums); + header.createAttribute>("NumPart_ThisFile", + HighFive::DataSpace::From(nums)).write(nums); { int n=0; for (auto c : comp->components) nums[n++] = c->CurTotal(); } - header.createAttribute>("NumPart_Total", HighFive::DataSpace::From(nums)).write(nums); + header.createAttribute>("NumPart_Total", + HighFive::DataSpace::From(nums)).write(nums); - header.createAttribute("Time", HighFive::DataSpace::From(tnow)).write(tnow); + header.createAttribute("Time", + HighFive::DataSpace::From(tnow)).write(tnow); // Create a new group for Parameters // From 724d43d43eb4277fc7771767fc8cb4f5b7257253 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 28 May 2025 19:48:34 -0400 Subject: [PATCH 048/106] A few additional tweaks; ready for testing --- src/OutHDF5.H | 9 ++- src/OutHDF5.cc | 147 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 107 insertions(+), 49 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index e26849334..003c93435 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -12,6 +12,7 @@ @param nbeg is suffix of the first phase space %dump @param timer set to true turns on wall-clock timer for PS output @param threads number of threads for binary writes + @param noids set to true turns off particle id writing */ class OutHDF5 : public Output @@ -20,13 +21,18 @@ class OutHDF5 : public Output private: std::string filename; - bool real4=true, real8=false, timer; + bool real4=true, real8=false, ids=true, timer; int nbeg, threads; void initialize(void); + std::vector masses; + std::vector multim; //! Valid keys for YAML configurations static const std::set valid_keys; + //! Check for single particle mass on the first invocation + void checkParticleMasses(); + public: //! Constructor @@ -41,6 +47,7 @@ public: indepentently of whether or not the frequency criterion is met \param timer set to true turns on wall-clock timer for PS output \param threads is the thread count for binary writes + \param noids set to true turns off particle id writing */ void Run(int nstep, int mstep, bool last); diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 071a388f8..5f6acfc80 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -20,6 +20,7 @@ OutHDF5::valid_keys = { "nint", "nintsub", "nbeg", + "noids", "real4", "real8", "timer", @@ -82,6 +83,10 @@ void OutHDF5::initialize() real4 = not real8; } + if (Output::conf["noids"]) { + ids = not Output::conf["noids"].as(); + } + if (Output::conf["timer"]) timer = Output::conf["timer"].as(); else @@ -152,8 +157,8 @@ void OutHDF5::Run(int n, int mstep, bool last) std::ostringstream fname; // Output name prefix - fname << filename << "." << setw(5) << setfill('0') << nbeg++ - << "." << myid+1; + fname << filename << "." << setw(5) << setfill('0') << nbeg++; + if (numprocs>1) fname << "." << myid+1; // Master file name std::string path = outdir + fname.str(); @@ -177,77 +182,90 @@ void OutHDF5::Run(int n, int mstep, bool last) HighFive::File::Create); try { - // Create a new group for Config - // - HighFive::Group config = file->createGroup("Config"); - - int ncomp = comp->components.size(); - config.createAttribute("NTYPES", - HighFive::DataSpace::From(ncomp)).write(ncomp); - - std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); - config.createAttribute("Git_commit", - HighFive::DataSpace::From(gcommit)).write(gcommit); - - config.createAttribute("Git_branch", - HighFive::DataSpace::From(gbranch)).write(gbranch); - - config.createAttribute("Compile_date", - HighFive::DataSpace::From(gdate)).write(gdate); - // Create a new group for Header // HighFive::Group header = file->createGroup("Header"); int dp = 1; - header.createAttribute("Flag_DoublePrecision", - HighFive::DataSpace::From(dp)).write(dp); + header.createAttribute + ("Flag_DoublePrecision", HighFive::DataSpace::From(dp)).write(dp); double hubble = 1, zero = 0; - header.createAttribute("HubbleParam", - HighFive::DataSpace::From(hubble)).write(hubble); + header.createAttribute + ("HubbleParam", HighFive::DataSpace::From(hubble)).write(hubble); - header.createAttribute("Omega0", - HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute + ("Omega0", HighFive::DataSpace::From(zero)).write(zero); - header.createAttribute("OmegaBaryon", - HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute + ("OmegaBaryon", HighFive::DataSpace::From(zero)).write(zero); - header.createAttribute("OmegaLambda", - HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute + ("OmegaLambda", HighFive::DataSpace::From(zero)).write(zero); - header.createAttribute("Redshift", - HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute + ("Redshift", HighFive::DataSpace::From(zero)).write(zero); - std::vector masses(ncomp, 0.0); - header.createAttribute>("MassTable", - HighFive::DataSpace::From(masses)).write(masses); + if (masses.size()==0) checkParticleMasses(); + header.createAttribute + >("MassTable", HighFive::DataSpace::From(masses)).write(masses); - header.createAttribute("NumFilesPerSnapshot", - HighFive::DataSpace::From(numprocs)).write(numprocs); + header.createAttribute + ("NumFilesPerSnapshot", HighFive::DataSpace::From(numprocs)).write(numprocs); - std::vector nums(ncomp); + std::vector nums(masses.size()); { int n=0; for (auto c : comp->components) nums[n++] = c->Number(); } - header.createAttribute>("NumPart_ThisFile", - HighFive::DataSpace::From(nums)).write(nums); + header.createAttribute> + ("NumPart_ThisFile", HighFive::DataSpace::From(nums)).write(nums); { int n=0; for (auto c : comp->components) nums[n++] = c->CurTotal(); } - header.createAttribute>("NumPart_Total", - HighFive::DataSpace::From(nums)).write(nums); + header.createAttribute> + ("NumPart_Total", HighFive::DataSpace::From(nums)).write(nums); - header.createAttribute("Time", - HighFive::DataSpace::From(tnow)).write(tnow); + header.createAttribute + ("Time", HighFive::DataSpace::From(tnow)).write(tnow); // Create a new group for Parameters // HighFive::Group params = file->createGroup("Parameters"); + std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); + params.createAttribute("Git_commit", + HighFive::DataSpace::From(gcommit)).write(gcommit); + + params.createAttribute("Git_branch", + HighFive::DataSpace::From(gbranch)).write(gbranch); + + params.createAttribute("Compile_date", + HighFive::DataSpace::From(gdate)).write(gdate); + + std::vector names, forces, configs; + for (auto c : comp->components) { + names.push_back(c->name); + forces.push_back(c->id); + YAML::Emitter out; + out << c->fconf; // where node is your YAML::Node + configs.push_back(out.c_str()); + } + + params.createAttribute + ("ComponentNames", + HighFive::DataSpace::From(names)).write(names); + + params.createAttribute + ("ForceMethods", + HighFive::DataSpace::From(forces)).write(forces); + + params.createAttribute + ("ForceConfigurations", + HighFive::DataSpace::From(configs)).write(configs); + } catch (HighFive::Exception& err) { - std::string msg("Coefs::factory: error reading HDF5 file, "); + std::string msg("Coefs::factory: error writing HDF5 file, "); throw std::runtime_error(msg + err.what()); } @@ -257,7 +275,7 @@ void OutHDF5::Run(int n, int mstep, bool last) nOK = 1; } - MPI_Allreduce(0, &nOK, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &nOK, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); // Exit on file open failure // @@ -285,9 +303,9 @@ void OutHDF5::Run(int n, int mstep, bool last) auto pgroup = file->createGroup(sout.str()); if (real4) - c->write_HDF5(pgroup, true, true); + c->write_HDF5(pgroup, multim[count], ids); else - c->write_HDF5(pgroup, true, true); + c->write_HDF5(pgroup, multim[count], ids); } chktimer.mark(); @@ -303,3 +321,36 @@ void OutHDF5::Run(int n, int mstep, bool last) } } +void OutHDF5::checkParticleMasses() +{ + for (auto c : comp->components) { + // First bunch of particles + int number = -1; + PartPtr *p = c->get_particles(&number); + + double minMass = std::numeric_limits::max(); + double maxMass = 0.0; + + // Keep going... + while (p) { + for (int k=0; kmass, minMass); + maxMass = std::max(P->mass, maxMass); + } + // Next bunch of particles + p = c->get_particles(&number); + } + + MPI_Allreduce(MPI_IN_PLACE, &minMass, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &maxMass, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + + if ( (maxMass - minMass)/maxMass < 1.0e-12) { + masses.push_back(maxMass); + multim.push_back(true); + } else { + masses.push_back(0.0); + multim.push_back(false); + } + } +} From 8ef1581423c0e5c1cfdad49dba10e71f3ea6ab06 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 29 May 2025 12:24:42 -0400 Subject: [PATCH 049/106] Implemented a HDF5 ParticleReader class; implemented compression; additional clean up --- exputil/ParticleReader.cc | 222 +++++++++++++++++++++++++++++++++++++- include/ParticleReader.H | 78 ++++++++++++++ src/Component.H | 6 ++ src/Component.cc | 66 ++++++------ src/OutHDF5.cc | 42 ++++++-- 5 files changed, 370 insertions(+), 44 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index f3aa05107..c67039fbd 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -7,7 +7,9 @@ #include #include #include - + // HighFive API +#include +#include #include // YAML support @@ -637,6 +639,219 @@ namespace PR { } + PSPhdf5::PSPhdf5(const std::vector& files, bool verbose) + { + _files = files; + _verbose = verbose; + + getInfo(); + curfile = _files.begin(); + + if (not nextFile()) { + std::cerr << "PSPhdf5: no files found" << std::endl; + } + } + + void PSPhdf5::getInfo() + { + // Read one file and get metadata + // + try { + // Silence the HDF5 error stack + // + HighFive::SilenceHDF5 quiet; + + // Try opening the file + // + HighFive::File h5file(_files[0], HighFive::File::ReadOnly); + + try { + + HighFive::Group header = h5file.getGroup("Header"); + HighFive::Group config = h5file.getGroup("Config"); + HighFive::Group params = h5file.getGroup("Parameters"); + + + // Get particle numbers + // + header.getAttribute("NumPart_Total").read(nptot); + totalCount = 0; + for (auto v : nptot) totalCount += v; + + // Get particle masses + // + header.getAttribute("MassTable").read(mass); + + // Get component names + // + params.getAttribute("ComponentNames").read(comps); + + // Get number of types + config.getAttribute("NTYPES").read(ntypes); + + config.getAttribute("Niattrib").read(Niattrib); + + config.getAttribute("Ndattrib").read(Ndattrib); + + // Real data type + // + int dp; + config.getAttribute("DOUBLEPRECISION").read(dp); + if (dp) real4 = false; + else real4 = true; + + } catch (HighFive::Exception& err) { + std::string msg("PSPhdf5: error reading HDF5 file, "); + throw std::runtime_error(msg + err.what()); + } + + } catch (HighFive::Exception& err) { + if (myid==0) + std::cerr << "---- Coefs::factory: " + << "error opening as HDF5, trying EXP native and ascii table" + << std::endl; + } + } + + bool PSPhdf5::nextFile() + { + if (curfile==_files.end()) return false; + if (real4) read_and_load(); + else read_and_load(); + curfile++; + return true; + } + + + template + void PSPhdf5::read_and_load() + { + // Try to catch and HDF5 and parsing errors + // + try { + // Silence the HDF5 error stack + // + HighFive::SilenceHDF5 quiet; + + // Try opening the file + // + HighFive::File h5file(_files[0], HighFive::File::ReadOnly); + + try { + + HighFive::Group header = h5file.getGroup("Header"); + + // Get time + // + header.getAttribute("Time").read(time); + + // Get particle counts + // + header.getAttribute("NumPart_ThisFile").read(npart); + + if (npart[curindx] > 0) { + // Make the group name + // + std::ostringstream sout; + sout << "PartType" << curindx; + + // Open the phase-space group + // + HighFive::Group part = h5file.getGroup(sout.str()); + + Eigen::Matrix pos, vel; + std::vector mas; + + Eigen::Matrix iattrib; + Eigen::Matrix rattrib; + + if (mass[curindx] > 0) + part.getDataSet("Masses").read(mas); + + part.getDataSet("Coordinates").read(pos); + part.getDataSet("Velocities" ).read(vel); + + if (Niattrib[curindx]>0) + part.getDataSet("IntAttributes" ).read(iattrib); + + if (Ndattrib[curindx]>0) + part.getDataSet("RealAttributes" ).read(rattrib); + + // Clear and load the particle vector + // + particles.clear(); + + Particle P; // Working particle will be copied + + // Load from the HDF5 datasets + // + for (int n=0; n 0) P.mass = mass[curindx]; + else P.mass = mas[n]; + + P.level = 0; + + for (int k=0; k<3; k++) { + P.pos[k] = pos(n, k); + P.vel[k] = vel(n, k); + } + + if (Niattrib[curindx]>0) { + P.iattrib.resize(Niattrib[curindx]); + for (int j=0; j0) { + P.iattrib.resize(Ndattrib[curindx]); + for (int j=0; j(); + template void PSPhdf5::read_and_load(); + + const Particle* PSPhdf5::firstParticle() + { + pcount = 0; + + return & particles[pcount++]; + } + + const Particle* PSPhdf5::nextParticle() + { + if (pcount < particles.size()) { + return & particles[pcount++]; + } else { + if (nextFile()) return firstParticle(); + else return 0; + } + } + + bool badstatus(istream& in) { ios::iostate i = in.rdstate(); @@ -1295,7 +1510,8 @@ namespace PR { std::vector ParticleReader::readerTypes - {"PSPout", "PSPspl", "GadgetNative", "GadgetHDF5", "TipsyNative", "TipsyXDR", "Bonsai"}; + {"PSPout", "PSPspl", "GadgetNative", "GadgetHDF5", "PSPhdf5", + "TipsyNative", "TipsyXDR", "Bonsai"}; std::vector> @@ -1373,6 +1589,8 @@ namespace PR { ret = std::make_shared(file, verbose); else if (reader.find("PSPspl") == 0) ret = std::make_shared(file, verbose); + else if (reader.find("PSPhdf5") == 0) + ret = std::make_shared(file, verbose); else if (reader.find("GadgetNative") == 0) ret = std::make_shared(file, verbose); else if (reader.find("GadgetHDF5") == 0) diff --git a/include/ParticleReader.H b/include/ParticleReader.H index 3ae70b174..625a87734 100644 --- a/include/ParticleReader.H +++ b/include/ParticleReader.H @@ -227,6 +227,84 @@ namespace PR }; + class PSPhdf5 : public ParticleReader + { + protected: + + //@{ + //! Parameters + double time; + std::vector particles; + bool _verbose; + std::vector _files; + + std::vector mass; + std::vector Niattrib, Ndattrib; + std::vector npart, nptot; + unsigned long totalCount; + int ntypes; + bool real4; + //@} + + //! Current file + std::vector::iterator curfile; + + //! Components + std::vector comps; + + //@{ + //! Current component and index + std::string curcomp; + int curindx; + //@} + + unsigned pcount; + template void read_and_load(); + void getInfo(); + void packParticle(); + bool nextFile(); + + public: + + //! Constructor + PSPhdf5(const std::vector& file, bool verbose=false); + + //! Select a particular particle type and reset the iterator + virtual void SelectType(const std::string& type) { + auto it = std::find(comps.begin(), comps.end(), type); + if (it != comps.end()) { + curcomp = type; + curindx = it - comps.begin(); + } + else { + std::cerr << "PSPhdf5 error: could not find particle component <" << type << ">" << std::endl; + std::cerr << "Available particle components are:"; + for (auto s : comps) std::cerr << " " << s; + std::cerr << std::endl; + throw std::runtime_error("PSPhdf5: non-existent component"); + } + + curfile = _files.begin(); // Set to first file and open + nextFile(); + } + + //! Number of particles in the chosen type + virtual unsigned long CurrentNumber() { return totalCount; } + + //! Return list of particle types + virtual std::vector GetTypes() { return comps; } + + //! Get current time + virtual double CurrentTime() { return time; } + + //! Reset to beginning of particles for this component + virtual const Particle* firstParticle(); + + //! Get the next particle + virtual const Particle* nextParticle(); + + }; + class PSPstanza { public: diff --git a/src/Component.H b/src/Component.H index 8f7adfab7..f38221a09 100644 --- a/src/Component.H +++ b/src/Component.H @@ -355,6 +355,12 @@ protected: //@} + //@{ + //! HDF5 compression parameters + int H5compress = 0; + int H5chunk = 4096; + //@} + public: //! Describe the phase space coordinates diff --git a/src/Component.cc b/src/Component.cc index 39a4ccc99..4b15bf712 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -86,7 +86,9 @@ const std::set Component::valid_keys_parm = "ctr_name", "noswitch", "freezeL", - "dtreset" + "dtreset", + "H5compress", + "H5chunk" }; const std::set Component::valid_keys_force = @@ -830,6 +832,9 @@ void Component::configure(void) if (cconf["noswitch"]) noswitch = cconf["noswitch"].as(); if (cconf["freezeL"]) freezeLev = cconf["freezeL" ].as(); if (cconf["dtreset"]) dtreset = cconf["dtreset" ].as(); + if (cconf["H5compress"]) + H5compress = cconf["H5compress"].as(); + if (cconf["H5chunk"]) H5chunk = cconf["H5chunk" ].as(); if (cconf["ton"]) { ton = cconf["ton"].as(); @@ -2251,24 +2256,19 @@ void Component::write_binary(ostream* out, bool real4) template void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) { - Eigen::Matrix pos(nbodies, 3); - Eigen::Matrix vel(nbodies, 3); + Eigen::Matrix pos(nbodies, 3); + Eigen::Matrix vel(nbodies, 3); std::vector mas; std::vector ids; - std::vector> iattrib; - std::vector> rattrib; + + Eigen::Matrix iattrib; + Eigen::Matrix dattrib; - if (masses) mas.resize(nbodies); - if (IDs) ids.resize(nbodies); - if (niattrib) { - iattrib.resize(niattrib); - for (auto & v : iattrib) v.resize(nbodies); - } - if (ndattrib) { - rattrib.resize(ndattrib); - for (auto & v : rattrib) v.resize(nbodies); - } + if (masses) mas.resize(nbodies); + if (IDs) ids.resize(nbodies); + if (niattrib) iattrib.resize(nbodies, niattrib); + if (ndattrib) dattrib.resize(nbodies, niattrib); unsigned int ctr = 0; @@ -2284,8 +2284,8 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) if (IDs) ids[ctr] = P->indx; for (int j=0; j<3; j++) pos(ctr, j) = p[k]->pos[j]; for (int j=0; j<3; j++) vel(ctr, j) = p[k]->vel[j]; - for (int j=0; jiattrib[j]; - for (int j=0; jdattrib[j]; + for (int j=0; jiattrib[j]; + for (int j=0; jdattrib[j]; // Increment array position ctr++; } @@ -2293,29 +2293,31 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) p = get_particles(&number); } + auto dcpl = HighFive::DataSetCreateProps{}; + + if (H5compress) { + dcpl.add(HighFive::Chunking(H5chunk, 1)); + dcpl.add(HighFive::Shuffle()); + dcpl.add(HighFive::Deflate(H5compress)); + } + // Add all datasets if (masses) { - HighFive::DataSet ds = group.createDataSet("Masses", mas); + HighFive::DataSet ds = group.createDataSet("Masses", mas, dcpl); } if (IDs) { - HighFive::DataSet ds = group.createDataSet("ParticleIDs", ids); + HighFive::DataSet ds = group.createDataSet("ParticleIDs", ids, dcpl); } - HighFive::DataSet dsPos = group.createDataSet("Positions", pos); - HighFive::DataSet dsVel = group.createDataSet("Velocities", vel); + HighFive::DataSet dsPos = group.createDataSet("Coordinates", pos, dcpl); + HighFive::DataSet dsVel = group.createDataSet("Velocities", vel); - if (int n=0; n0) + HighFive::DataSet dsInt = group.createDataSet("IntAttributes", iattrib, dcpl); + + if (ndattrib>0) + HighFive::DataSet dsReal = group.createDataSet("RealAttributes", dattrib, dcpl); } // Explicit instantiations for float and double diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 5f6acfc80..6b673ca0b 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -4,10 +4,9 @@ #include #include -#include -#include -#include -#include +// HighFive API for HDF5 +#include +#include #include "expand.H" #include @@ -212,23 +211,46 @@ void OutHDF5::Run(int n, int mstep, bool last) header.createAttribute ("NumFilesPerSnapshot", HighFive::DataSpace::From(numprocs)).write(numprocs); - std::vector nums(masses.size()); + std::vector nums(masses.size()); { int n=0; for (auto c : comp->components) nums[n++] = c->Number(); } - header.createAttribute> + header.createAttribute> ("NumPart_ThisFile", HighFive::DataSpace::From(nums)).write(nums); { int n=0; for (auto c : comp->components) nums[n++] = c->CurTotal(); } - header.createAttribute> + header.createAttribute> ("NumPart_Total", HighFive::DataSpace::From(nums)).write(nums); header.createAttribute ("Time", HighFive::DataSpace::From(tnow)).write(tnow); + // Create a new group for Config + // + HighFive::Group config = file->createGroup("Config"); + + int ntypes = comp->components.size(); + config.createAttribute + ("NTYPES", HighFive::DataSpace::From(ntypes)).write(ntypes); + + config.createAttribute + ("DOUBLEPRECISION", HighFive::DataSpace::From(dp)).write(dp); + + std::vector Niattrib, Ndattrib; + for (auto & c : comp->components) { + Niattrib.push_back(c->niattrib); + Ndattrib.push_back(c->ndattrib); + } + + config.createAttribute> + ("Niattrib", HighFive::DataSpace::From(Niattrib)).write(Niattrib); + + config.createAttribute> + ("Ndattrib", HighFive::DataSpace::From(Ndattrib)).write(Ndattrib); + // Create a new group for Parameters // HighFive::Group params = file->createGroup("Parameters"); @@ -252,15 +274,15 @@ void OutHDF5::Run(int n, int mstep, bool last) configs.push_back(out.c_str()); } - params.createAttribute + params.createAttribute> ("ComponentNames", HighFive::DataSpace::From(names)).write(names); - params.createAttribute + params.createAttribute> ("ForceMethods", HighFive::DataSpace::From(forces)).write(forces); - params.createAttribute + params.createAttribute> ("ForceConfigurations", HighFive::DataSpace::From(configs)).write(configs); From 6ed1587a08d09578e70e68149e37edf17fc32cee Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 29 May 2025 15:40:43 -0400 Subject: [PATCH 050/106] Fix a few mistakes in logic; add a default particle type --- exputil/ParticleReader.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index c67039fbd..1ff822c1e 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -686,7 +686,12 @@ namespace PR { // params.getAttribute("ComponentNames").read(comps); + // Set some defaults + curcomp = comps[0]; + curindx = 0; + // Get number of types + // config.getAttribute("NTYPES").read(ntypes); config.getAttribute("Niattrib").read(Niattrib); @@ -765,7 +770,7 @@ namespace PR { Eigen::Matrix iattrib; Eigen::Matrix rattrib; - if (mass[curindx] > 0) + if (mass[curindx] == 0) part.getDataSet("Masses").read(mas); part.getDataSet("Coordinates").read(pos); From 1765790bb17ab1e3331e12bb7fc75308a86d434d Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 29 May 2025 15:41:30 -0400 Subject: [PATCH 051/106] Add some data compression --- src/Component.cc | 81 ++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/src/Component.cc b/src/Component.cc index 4b15bf712..e4f3d6eb1 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -832,9 +832,6 @@ void Component::configure(void) if (cconf["noswitch"]) noswitch = cconf["noswitch"].as(); if (cconf["freezeL"]) freezeLev = cconf["freezeL" ].as(); if (cconf["dtreset"]) dtreset = cconf["dtreset" ].as(); - if (cconf["H5compress"]) - H5compress = cconf["H5compress"].as(); - if (cconf["H5chunk"]) H5chunk = cconf["H5chunk" ].as(); if (cconf["ton"]) { ton = cconf["ton"].as(); @@ -2252,7 +2249,8 @@ void Component::write_binary(ostream* out, bool real4) } -//! Write HDF5 phase-space structure +//! Write HDF5 phase-space structure for this nodes particles only. +//! Use template to specialize to either float or double precision. template void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) { @@ -2271,53 +2269,68 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) if (ndattrib) dattrib.resize(nbodies, niattrib); unsigned int ctr = 0; - - // First bunch of particles - int number = -1; - PartPtr *p = get_particles(&number); - - // Keep going... - while (p) { - for (int k=0; kmass; - if (IDs) ids[ctr] = P->indx; - for (int j=0; j<3; j++) pos(ctr, j) = p[k]->pos[j]; - for (int j=0; j<3; j++) vel(ctr, j) = p[k]->vel[j]; - for (int j=0; jiattrib[j]; - for (int j=0; jdattrib[j]; - // Increment array position - ctr++; - } - // Next bunch of particles - p = get_particles(&number); + for (auto it=particles.begin(); it!=particles.end(); it++) { + + // Get the particle pointer + auto &P = it->second; + + // Pack HDF5 data + if (masses) mas[ctr] = P->mass; + if (IDs) ids[ctr] = P->indx; + for (int j=0; j<3; j++) pos(ctr, j) = P->pos[j]; + for (int j=0; j<3; j++) vel(ctr, j) = P->vel[j]; + for (int j=0; jiattrib[j]; + for (int j=0; jdattrib[j]; + + // Increment array position + ctr++; } - auto dcpl = HighFive::DataSetCreateProps{}; + // Set properties, if requested + auto dcpl1 = HighFive::DataSetCreateProps{}; + auto dcpl3 = HighFive::DataSetCreateProps{}; + auto dcplI = HighFive::DataSetCreateProps{}; + auto dcplD = HighFive::DataSetCreateProps{}; if (H5compress) { - dcpl.add(HighFive::Chunking(H5chunk, 1)); - dcpl.add(HighFive::Shuffle()); - dcpl.add(HighFive::Deflate(H5compress)); + int chunk = H5chunk; + + // Sanity + if (H5chunk >= nbodies) { + chunk = nbodies/8; + } + + dcpl1.add(HighFive::Chunking(chunk)); + dcpl1.add(HighFive::Shuffle()); + dcpl1.add(HighFive::Deflate(H5compress)); + + dcpl3.add(HighFive::Chunking(chunk, 3)); + dcpl3.add(HighFive::Shuffle()); + dcpl3.add(HighFive::Deflate(H5compress)); + + if (niattrib) dcplI.add(HighFive::Chunking(chunk, niattrib)); + if (ndattrib) dcplD.add(HighFive::Chunking(chunk, ndattrib)); + + std::cout << "Using compression=" << H5compress << " with chunk size " << chunk << std::endl; } // Add all datasets if (masses) { - HighFive::DataSet ds = group.createDataSet("Masses", mas, dcpl); + HighFive::DataSet ds = group.createDataSet("Masses", mas, dcpl1); } if (IDs) { - HighFive::DataSet ds = group.createDataSet("ParticleIDs", ids, dcpl); + HighFive::DataSet ds = group.createDataSet("ParticleIDs", ids, dcpl1); } - HighFive::DataSet dsPos = group.createDataSet("Coordinates", pos, dcpl); - HighFive::DataSet dsVel = group.createDataSet("Velocities", vel); + HighFive::DataSet dsPos = group.createDataSet("Coordinates", pos, dcpl3); + HighFive::DataSet dsVel = group.createDataSet("Velocities", vel, dcpl3); if (niattrib>0) - HighFive::DataSet dsInt = group.createDataSet("IntAttributes", iattrib, dcpl); + HighFive::DataSet dsInt = group.createDataSet("IntAttributes", iattrib, dcplI); if (ndattrib>0) - HighFive::DataSet dsReal = group.createDataSet("RealAttributes", dattrib, dcpl); + HighFive::DataSet dsReal = group.createDataSet("RealAttributes", dattrib, dcplD); } // Explicit instantiations for float and double From 8e98d5bed9edcea6e5140ee98e67afaf70519f95 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 29 May 2025 15:41:43 -0400 Subject: [PATCH 052/106] Add some data compression --- src/Component.H | 9 +++++ src/OutHDF5.cc | 92 ++++++++++++++++++++----------------------------- 2 files changed, 47 insertions(+), 54 deletions(-) diff --git a/src/Component.H b/src/Component.H index f38221a09..56aa148f0 100644 --- a/src/Component.H +++ b/src/Component.H @@ -986,6 +986,15 @@ public: //! Compute level from minimum requested time step from last master step inline bool DTreset() { return dtreset; } + //! Set H5 compression parameters; in principle, compression can be + //! set per component, although it is set to be the same for all + //! components currently. + void setH5(int compress, int chunk) + { + H5compress = compress; + H5chunk = chunk; + } + #if HAVE_LIBCUDA==1 //@{ //! CUDA utilities for handling host <===> device exchange diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 6b673ca0b..d641456f4 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -23,7 +23,9 @@ OutHDF5::valid_keys = { "real4", "real8", "timer", - "threads" + "threads," + "H5compress", + "H5chunk" }; OutHDF5::OutHDF5(const YAML::Node& conf) : Output(conf) @@ -95,6 +97,16 @@ void OutHDF5::initialize() threads = Output::conf["threads"].as(); else threads = 0; + + int H5compress = 0, H5chunk = 0; + if (Output::conf["H5compress"]) + H5compress = Output::conf["H5compress"].as(); + + if (Output::conf["H5chunk"]) + H5chunk = Output::conf["H5chunk"].as(); + + if (H5chunk>0) + for (auto c : comp->components) c->setH5(H5compress, H5chunk); } catch (YAML::Exception & error) { if (myid==0) std::cout << "Error parsing parameters in OutHDF5: " @@ -156,7 +168,7 @@ void OutHDF5::Run(int n, int mstep, bool last) std::ostringstream fname; // Output name prefix - fname << filename << "." << setw(5) << setfill('0') << nbeg++; + fname << filename << "_" << setw(5) << setfill('0') << nbeg++; if (numprocs>1) fname << "." << myid+1; // Master file name @@ -185,59 +197,46 @@ void OutHDF5::Run(int n, int mstep, bool last) // HighFive::Group header = file->createGroup("Header"); int dp = 1; - header.createAttribute - ("Flag_DoublePrecision", HighFive::DataSpace::From(dp)).write(dp); + header.createAttribute("Flag_DoublePrecision", dp); double hubble = 1, zero = 0; - header.createAttribute - ("HubbleParam", HighFive::DataSpace::From(hubble)).write(hubble); + header.createAttribute("HubbleParam", hubble); - header.createAttribute - ("Omega0", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("Omega0", zero); - header.createAttribute - ("OmegaBaryon", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("OmegaBaryon", zero); - header.createAttribute - ("OmegaLambda", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("OmegaLambda", zero); - header.createAttribute - ("Redshift", HighFive::DataSpace::From(zero)).write(zero); + header.createAttribute("Redshift", zero); if (masses.size()==0) checkParticleMasses(); - header.createAttribute - >("MassTable", HighFive::DataSpace::From(masses)).write(masses); + header.createAttribute("MassTable", masses); - header.createAttribute - ("NumFilesPerSnapshot", HighFive::DataSpace::From(numprocs)).write(numprocs); + header.createAttribute("NumFilesPerSnapshot", numprocs); std::vector nums(masses.size()); { int n=0; for (auto c : comp->components) nums[n++] = c->Number(); } - header.createAttribute> - ("NumPart_ThisFile", HighFive::DataSpace::From(nums)).write(nums); + header.createAttribute("NumPart_ThisFile", nums); { int n=0; for (auto c : comp->components) nums[n++] = c->CurTotal(); } - header.createAttribute> - ("NumPart_Total", HighFive::DataSpace::From(nums)).write(nums); + header.createAttribute("NumPart_Total", nums); - header.createAttribute - ("Time", HighFive::DataSpace::From(tnow)).write(tnow); + header.createAttribute("Time", tnow); // Create a new group for Config // HighFive::Group config = file->createGroup("Config"); int ntypes = comp->components.size(); - config.createAttribute - ("NTYPES", HighFive::DataSpace::From(ntypes)).write(ntypes); + config.createAttribute("NTYPES", ntypes); - config.createAttribute - ("DOUBLEPRECISION", HighFive::DataSpace::From(dp)).write(dp); + config.createAttribute("DOUBLEPRECISION", dp); std::vector Niattrib, Ndattrib; for (auto & c : comp->components) { @@ -245,25 +244,18 @@ void OutHDF5::Run(int n, int mstep, bool last) Ndattrib.push_back(c->ndattrib); } - config.createAttribute> - ("Niattrib", HighFive::DataSpace::From(Niattrib)).write(Niattrib); + config.createAttribute("Niattrib", Niattrib); - config.createAttribute> - ("Ndattrib", HighFive::DataSpace::From(Ndattrib)).write(Ndattrib); + config.createAttribute("Ndattrib", Ndattrib); // Create a new group for Parameters // HighFive::Group params = file->createGroup("Parameters"); std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); - params.createAttribute("Git_commit", - HighFive::DataSpace::From(gcommit)).write(gcommit); - - params.createAttribute("Git_branch", - HighFive::DataSpace::From(gbranch)).write(gbranch); - - params.createAttribute("Compile_date", - HighFive::DataSpace::From(gdate)).write(gdate); + params.createAttribute("Git_commit", gcommit); + params.createAttribute("Git_branch", gbranch); + params.createAttribute("Compile_date", gdate); std::vector names, forces, configs; for (auto c : comp->components) { @@ -274,20 +266,12 @@ void OutHDF5::Run(int n, int mstep, bool last) configs.push_back(out.c_str()); } - params.createAttribute> - ("ComponentNames", - HighFive::DataSpace::From(names)).write(names); - - params.createAttribute> - ("ForceMethods", - HighFive::DataSpace::From(forces)).write(forces); - - params.createAttribute> - ("ForceConfigurations", - HighFive::DataSpace::From(configs)).write(configs); + params.createAttribute("ComponentNames", names); + params.createAttribute("ForceMethods", forces); + params.createAttribute("ForceConfigurations", configs); } catch (HighFive::Exception& err) { - std::string msg("Coefs::factory: error writing HDF5 file, "); + std::string msg("OutHDF5: error writing HDF5 file, "); throw std::runtime_error(msg + err.what()); } @@ -364,8 +348,8 @@ void OutHDF5::checkParticleMasses() p = c->get_particles(&number); } - MPI_Allreduce(MPI_IN_PLACE, &minMass, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, &maxMass, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Bcast(&minMass, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); + MPI_Bcast(&maxMass, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); if ( (maxMass - minMass)/maxMass < 1.0e-12) { masses.push_back(maxMass); From 15ce5812c7f15a25e45951cd8d35f4202423cf30 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 29 May 2025 16:24:33 -0400 Subject: [PATCH 053/106] Removed debug output --- src/Component.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Component.cc b/src/Component.cc index e4f3d6eb1..8b96dba3b 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2310,8 +2310,6 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) if (niattrib) dcplI.add(HighFive::Chunking(chunk, niattrib)); if (ndattrib) dcplD.add(HighFive::Chunking(chunk, ndattrib)); - - std::cout << "Using compression=" << H5compress << " with chunk size " << chunk << std::endl; } // Add all datasets From 5539761d5f68d26f553e135b4ac163dba3272ae6 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 31 May 2025 13:27:24 -0400 Subject: [PATCH 054/106] Added native-style PSP HDF5; this is now the default --- exputil/ParticleReader.cc | 285 +++++++++++++++++++++++++++++++++++++- include/ParticleReader.H | 5 +- src/Component.H | 18 ++- src/Component.cc | 131 +++++++++++++++++- src/OutHDF5.H | 11 +- src/OutHDF5.cc | 251 +++++++++++++++++++++++++++++++-- 6 files changed, 673 insertions(+), 28 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 1ff822c1e..15e55c1fe 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -8,6 +8,7 @@ #include #include // HighFive API +#include #include #include @@ -671,7 +672,6 @@ namespace PR { HighFive::Group config = h5file.getGroup("Config"); HighFive::Group params = h5file.getGroup("Parameters"); - // Get particle numbers // header.getAttribute("NumPart_Total").read(nptot); @@ -687,9 +687,16 @@ namespace PR { params.getAttribute("ComponentNames").read(comps); // Set some defaults + // curcomp = comps[0]; curindx = 0; + // Get packing type (PSP or Gadget4; PSP is default) + // + int style; + config.getAttribute("PSPstyle").read(style); + gadget4 = (style == 0); + // Get number of types // config.getAttribute("NTYPES").read(ntypes); @@ -712,7 +719,7 @@ namespace PR { } catch (HighFive::Exception& err) { if (myid==0) - std::cerr << "---- Coefs::factory: " + std::cerr << "---- PSPhdf5: " << "error opening as HDF5, trying EXP native and ascii table" << std::endl; } @@ -730,6 +737,13 @@ namespace PR { template void PSPhdf5::read_and_load() + { + if (gadget4) read_and_load_gadget4(); + else read_and_load_psp(); + } + + template + void PSPhdf5::read_and_load_gadget4() { // Try to catch and HDF5 and parsing errors // @@ -740,7 +754,7 @@ namespace PR { // Try opening the file // - HighFive::File h5file(_files[0], HighFive::File::ReadOnly); + HighFive::File h5file(*curfile, HighFive::File::ReadOnly); try { @@ -821,23 +835,277 @@ namespace PR { } // END: we have particles - } catch (HighFive::Exception& err) { + } catch (HighFive::Exception & err) { std::string msg("PSPhdf5: error reading HDF5 file, "); throw std::runtime_error(msg + err.what()); } - } catch (HighFive::Exception& err) { + } catch (HighFive::Exception & err) { if (myid==0) - std::cerr << "---- Coefs::factory: " + std::cerr << "---- PSPhdf5 gadget-4 read error: " << "error opening as HDF5, trying EXP native and ascii table" << std::endl; } } - // Template specializations + // Helper structure + template + struct H5Particle + { + unsigned long id; + T mass; + T pos[3]; + T vel[3]; + hvl_t iattrib; + hvl_t dattrib; + }; + + // Read and return a scalar HDF5 attribute + template + T readScalar(H5::Group& group, const std::string& name) + { + H5::DataType type; + + // Create a int datatype + if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_INT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_ULONG; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_FLOAT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_DOUBLE; + } + else if constexpr (std::is_same_v) { + type = H5::StrType(0, H5T_VARIABLE); + } + else { + assert(0); + } + + // Create the attribute + H5::Attribute attribute = group.openAttribute(name); + + T ret; + + // Treat string separately + if constexpr (std::is_same_v) { + attribute.read(type, &ret); + } else + attribute.read(type, &ret); + + return ret; + }; + + + // Read and return a std::vector HDF5 attribute + template + std::vector readVector(H5::Group& group, const std::string& name) + { + H5::DataType type; + + // Create a int datatype + if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_INT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_ULONG; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_FLOAT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_DOUBLE; + } + else if constexpr (std::is_same_v) { + type = H5::StrType(0, H5T_VARIABLE); + } + else { + assert(0); + } + + // Create the attribute + H5::Attribute attribute = group.openAttribute(name); + H5::DataType attributeType = attribute.getDataType(); + hsize_t attributeSize = attribute.getSpace().getSimpleExtentNpoints(); + + // Create the return value + std::vector ret(attributeSize); + + + // Treat string separately + if constexpr (std::is_same_v) { + // Allocate memory for string pointers + std::vector stringPointers(attributeSize); + + // Read the string data + attribute.read(attributeType, stringPointers.data()); + + // Create std::vector + for (int i = 0; i < attributeSize; ++i) { + ret.emplace_back(stringPointers[i]); + } + + // Release memory allocated by HDF5 + H5::DataSet::vlenReclaim(stringPointers.data(), attributeType, attribute.getSpace()); + } else + attribute.read(attributeType, ret.data()); + + return ret; + }; + + + template + void PSPhdf5::read_and_load_psp() + { + // Try to catch and HDF5 and parsing errors + // + try { + // Try opening the file + // + H5::H5File file(*curfile, H5F_ACC_RDONLY); + + try { + + // Define the compound datatype + // + H5::CompType compound_type(sizeof(H5Particle)); + + H5::DataType type; + if constexpr (std::is_same_v) + type = H5::PredType::NATIVE_DOUBLE; + else if constexpr (std::is_same_v) + type = H5::PredType::NATIVE_FLOAT; + else assert(0); + + // Add members + compound_type.insertMember("id", HOFFSET(H5Particle, id), H5::PredType::NATIVE_INT); + compound_type.insertMember("mass", HOFFSET(H5Particle, mass), type); + + hsize_t dims3[1] = {3}; + H5::ArrayType array3_type(type, 1, dims3); + + compound_type.insertMember("pos", HOFFSET(H5Particle, pos), array3_type); + compound_type.insertMember("vel", HOFFSET(H5Particle, vel), array3_type); + compound_type.insertMember("iattrib", HOFFSET(H5Particle, iattrib), H5::VarLenType(H5::PredType::NATIVE_INT)); + compound_type.insertMember("dattrib", HOFFSET(H5Particle, dattrib), H5::VarLenType(type)); + + // Open the particle group + // + H5::Group header = file.openGroup("Header"); + + // Get time + // + time = readScalar(header, "Time"); + + // Get particle counts + // + npart = readVector(header, "NumPart_ThisFile"); + + if (npart[curindx] > 0) { + // Make the group name + // + std::ostringstream sout; + sout << "PartType" << curindx; + + // Open the phase-space group + // + H5::Group part = file.openGroup(sout.str()); + + // Clear and load the particle vector + // + particles.clear(); + + // Working particle; will be copied to particles array + // + Particle P; + + // Get the particle dataset from HDF5 + // + H5::DataSet dataset = part.openDataSet("particles"); + H5::DataSpace dataspace = dataset.getSpace(); + + // Sanity check + // + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims); + size_t num_elements = dims[0]; + + if (num_elements != npart[curindx]) + throw std::runtime_error("PSPhdf5: number of particles in file does not match the number in header"); + + // Read particles + // + std::vector> h5part(num_elements); + dataset.read(h5part.data(), compound_type); + + // Load from the HDF5 dataset + // + for (int n=0; n0) { + P.iattrib.resize(Niattrib[curindx]); + // Access data by pointer + int* ptr = (int*)h5part[n].iattrib.p; + for (int j=0; j0) { + P.iattrib.resize(Ndattrib[curindx]); + // Access data by pointer + Scalar* ptr = (Scalar*)h5part[n].dattrib.p; + for (int j=0; j(); template void PSPhdf5::read_and_load(); + template void PSPhdf5::read_and_load_gadget4(); + template void PSPhdf5::read_and_load_gadget4(); + template void PSPhdf5::read_and_load_psp(); + template void PSPhdf5::read_and_load_psp(); const Particle* PSPhdf5::firstParticle() { @@ -1580,6 +1848,9 @@ namespace PR { } } + // Final batch + if (batch.size()) batches.push_back(batch); + return batches; } diff --git a/include/ParticleReader.H b/include/ParticleReader.H index 625a87734..a01970305 100644 --- a/include/ParticleReader.H +++ b/include/ParticleReader.H @@ -242,8 +242,8 @@ namespace PR std::vector Niattrib, Ndattrib; std::vector npart, nptot; unsigned long totalCount; + bool real4, gadget4; int ntypes; - bool real4; //@} //! Current file @@ -260,6 +260,9 @@ namespace PR unsigned pcount; template void read_and_load(); + template void read_and_load_gadget4(); + template void read_and_load_psp(); + void getInfo(); void packParticle(); bool nextFile(); diff --git a/src/Component.H b/src/Component.H index 56aa148f0..80aa8f77b 100644 --- a/src/Component.H +++ b/src/Component.H @@ -6,6 +6,8 @@ #include +#include + #include #include #include @@ -357,8 +359,9 @@ protected: //@{ //! HDF5 compression parameters - int H5compress = 0; - int H5chunk = 4096; + int H5compress = 0; + bool H5shuffle = true; + int H5chunk = 4096; //@} public: @@ -563,10 +566,18 @@ public: //! Write binary component phase-space structure void write_binary(ostream *out, bool real4 = false); + //@{ //! Write HDF5 phase-space structure for real or float + + //! Gadget style template void write_HDF5(HighFive::Group& group, bool masses, bool IDs); + //! PSP style + template + void write_H5(H5::Group& group); + //@} + //! Write header for per-node writes void write_binary_header(ostream* out, bool real4, std::string prefix, int nth=1); @@ -989,9 +1000,10 @@ public: //! Set H5 compression parameters; in principle, compression can be //! set per component, although it is set to be the same for all //! components currently. - void setH5(int compress, int chunk) + void setH5(int compress, bool shuffle, int chunk) { H5compress = compress; + H5shuffle = shuffle; H5chunk = chunk; } diff --git a/src/Component.cc b/src/Component.cc index 8b96dba3b..59a810c34 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -88,7 +88,8 @@ const std::set Component::valid_keys_parm = "freezeL", "dtreset", "H5compress", - "H5chunk" + "H5shuffle", + "H5chunk", }; const std::set Component::valid_keys_force = @@ -2301,15 +2302,23 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) } dcpl1.add(HighFive::Chunking(chunk)); - dcpl1.add(HighFive::Shuffle()); + if (H5shuffle) dcpl1.add(HighFive::Shuffle()); dcpl1.add(HighFive::Deflate(H5compress)); dcpl3.add(HighFive::Chunking(chunk, 3)); - dcpl3.add(HighFive::Shuffle()); + if (H5shuffle) dcpl3.add(HighFive::Shuffle()); dcpl3.add(HighFive::Deflate(H5compress)); - if (niattrib) dcplI.add(HighFive::Chunking(chunk, niattrib)); - if (ndattrib) dcplD.add(HighFive::Chunking(chunk, ndattrib)); + if (niattrib) { + dcplI.add(HighFive::Chunking(chunk, niattrib)); + if (H5shuffle) dcplI.add(HighFive::Shuffle()); + dcplI.add(HighFive::Deflate(H5compress)); + } + if (ndattrib) { + dcplD.add(HighFive::Chunking(chunk, ndattrib)); + if (H5shuffle) dcplD.add(HighFive::Shuffle()); + dcplD.add(HighFive::Deflate(H5compress)); + } } // Add all datasets @@ -2340,6 +2349,118 @@ template void Component::write_HDF5 (HighFive::Group& group, bool masses, bool IDs); +// Helper structure +template +struct H5Particle +{ + unsigned long id; + T mass; + T pos[3]; + T vel[3]; + hvl_t iattrib; + hvl_t dattrib; +}; + + +//! Write HDF5 phase-space structure for this nodes particles only. +template +void Component::write_H5(H5::Group& group) +{ + try { + + // Define the compound datatype + H5::CompType compound_type(sizeof(H5Particle)); + + H5::DataType type; + if constexpr (std::is_same_v) + type = H5::PredType::NATIVE_DOUBLE; + else if constexpr (std::is_same_v) + type = H5::PredType::NATIVE_FLOAT; + else assert(0); + + // Add members to the type + compound_type.insertMember("id", HOFFSET(H5Particle, id), H5::PredType::NATIVE_INT); + compound_type.insertMember("mass", HOFFSET(H5Particle, mass), type); + + hsize_t dims3[1] = {3}; + H5::ArrayType array3_type(type, 1, dims3); + + compound_type.insertMember("pos", HOFFSET(H5Particle, pos), array3_type); + compound_type.insertMember("vel", HOFFSET(H5Particle, vel), array3_type); + compound_type.insertMember("iattrib", HOFFSET(H5Particle, iattrib), H5::VarLenType(H5::PredType::NATIVE_INT)); + compound_type.insertMember("dattrib", HOFFSET(H5Particle, dattrib), H5::VarLenType(type)); + + + // Create the data space + std::vector> h5_particles(particles.size()); + + size_t n = 0; + for (auto it=particles.begin(); it!=particles.end(); it++, n++) { + + // Get the particle pointer + auto &P = it->second; + + // Load the data + h5_particles[n].id = P->indx; + h5_particles[n].mass = P->mass; + for (int k=0; k<3; k++) { + h5_particles[n].pos[k] = P->pos[k]; + h5_particles[n].vel[k] = P->vel[k]; + } + h5_particles[n].iattrib.len = P->iattrib.size(); + h5_particles[n].dattrib.len = P->dattrib.size(); + h5_particles[n].iattrib.p = P->iattrib.data(); + h5_particles[n].dattrib.p = P->dattrib.data(); + } + + // Set properties, if requested + H5::DSetCreatPropList dcpl; + + // This could be generalized by registering a user filter, like + // blosc. Right now, we're using the default (which is gzip) + if (H5compress) { + int chunk = H5chunk; + + // Sanity + if (H5chunk >= nbodies) { + chunk = nbodies/8; + } + + // Set chunking + hsize_t chunk_dims[1] = {chunk}; + dcpl.setChunk(1, chunk_dims); + + // Set compression level + dcpl.setDeflate(H5compress); + + // Enable shuffle filter + dcpl.setShuffle(); + } + + // Create dataspace + hsize_t dims[1] = {h5_particles.size()}; + H5::DataSpace dataspace(1, dims); + + // Create dataset + H5::DataSet dataset = group.createDataSet("particles", compound_type, dataspace, dcpl); + + // Write data to the dataset + dataset.write(h5_particles.data(), compound_type); + + } catch (H5::Exception &error) { + throw std::runtime_error("Component::writeH5 error: " + error.getDetailMsg()); + } + +} + +// Explicit instantiations for float and double +template +void Component::write_H5(H5::Group& group); + +template +void Component::write_H5(H5::Group& group); + + void Component::write_binary_header(ostream* out, bool real4, const std::string prefix, int nth) { ComponentHeader header; diff --git a/src/OutHDF5.H b/src/OutHDF5.H index 003c93435..d3d256141 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -12,6 +12,7 @@ @param nbeg is suffix of the first phase space %dump @param timer set to true turns on wall-clock timer for PS output @param threads number of threads for binary writes + @param gadget set to true for Gadget-4-type HDF5 @param noids set to true turns off particle id writing */ @@ -21,7 +22,7 @@ class OutHDF5 : public Output private: std::string filename; - bool real4=true, real8=false, ids=true, timer; + bool real4=true, real8=false, ids=true, gadget4=false, timer; int nbeg, threads; void initialize(void); std::vector masses; @@ -33,6 +34,13 @@ private: //! Check for single particle mass on the first invocation void checkParticleMasses(); + //! Gadget-4 format + void RunGadget4(const std::string& path); + + //! PSP format + void RunPSP(const std::string& path); + + public: //! Constructor @@ -48,6 +56,7 @@ public: \param timer set to true turns on wall-clock timer for PS output \param threads is the thread count for binary writes \param noids set to true turns off particle id writing + \param gadget set to true for Gadget-4-type HDF5 */ void Run(int nstep, int mstep, bool last); diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index d641456f4..7492a8521 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -5,6 +6,7 @@ #include // HighFive API for HDF5 +#include #include #include @@ -24,6 +26,7 @@ OutHDF5::valid_keys = { "real8", "timer", "threads," + "gadget", "H5compress", "H5chunk" }; @@ -98,15 +101,29 @@ void OutHDF5::initialize() else threads = 0; + if (Output::conf["gadget"]) + gadget4 = Output::conf["gadget"].as(); + else + gadget4 = false; + + // Default HDF5 compression is no compression. By default, + // shuffle is on unless turned off manually. + // int H5compress = 0, H5chunk = 0; + bool H5shuffle= true; + if (Output::conf["H5compress"]) H5compress = Output::conf["H5compress"].as(); + if (Output::conf["H5shuffle"]) + H5shuffle = Output::conf["H5shuffle"].as(); + if (Output::conf["H5chunk"]) H5chunk = Output::conf["H5chunk"].as(); + // Register compression parameters win the Component instance if (H5chunk>0) - for (auto c : comp->components) c->setH5(H5compress, H5chunk); + for (auto c : comp->components) c->setH5(H5compress, H5shuffle, H5chunk); } catch (YAML::Exception & error) { if (myid==0) std::cout << "Error parsing parameters in OutHDF5: " @@ -174,6 +191,24 @@ void OutHDF5::Run(int n, int mstep, bool last) // Master file name std::string path = outdir + fname.str(); + if (gadget4) RunGadget4(path); + else RunPSP(path); + + chktimer.mark(); + + dump_signal = 0; + + if (timer) { + end = std::chrono::high_resolution_clock::now(); + std::chrono::duration intvl = end - beg; + if (myid==0) + std::cout << "OutHDF5 [T=" << tnow << "] timing=" << intvl.count() + << std::endl; + } +} + +void OutHDF5::RunGadget4(const std::string& path) +{ int nOK = 0; std::unique_ptr file; @@ -233,6 +268,9 @@ void OutHDF5::Run(int n, int mstep, bool last) // HighFive::Group config = file->createGroup("Config"); + int style = 0; + config.createAttribute("PSPstyle", style); + int ntypes = comp->components.size(); config.createAttribute("NTYPES", ntypes); @@ -290,10 +328,9 @@ void OutHDF5::Run(int n, int mstep, bool last) exit(33); } - int count = 0; for (auto c : comp->components) { - + #ifdef HAVE_LIBCUDA if (use_cuda) { if (c->force->cudaAware() and not comp->fetched[c]) { @@ -313,17 +350,209 @@ void OutHDF5::Run(int n, int mstep, bool last) else c->write_HDF5(pgroup, multim[count], ids); } +} - chktimer.mark(); - dump_signal = 0; +// Template to write a scalar attribute for types used here +template +void writeScalar(H5::Group& group, const std::string& name, T value) + { + // Create a dataspace for the attribute (scalar in this case) + H5::DataSpace space(H5S_SCALAR); + + H5::DataType type; - if (timer) { - end = std::chrono::high_resolution_clock::now(); - std::chrono::duration intvl = end - beg; - if (myid==0) - std::cout << "OutHDF5 [T=" << tnow << "] timing=" << intvl.count() - << std::endl; + // Create a int datatype + if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_INT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_ULONG; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_FLOAT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_DOUBLE; + } + else if constexpr (std::is_same_v) { + type = H5::StrType(0, H5T_VARIABLE); + } + else { + assert(0); + } + + // Create the attribute + H5::Attribute attribute = group.createAttribute(name, type, space); + + // Treat string separately + if constexpr (std::is_same_v) { + const char* strPtr = value.c_str(); + attribute.write(type, &strPtr); + } else + attribute.write(type, &value); + }; + + +// Template to write a std::vector attribute for types used here +template +void writeVector(H5::Group& group, const std::string& name, std::vector& value) + { + // Create a dataspace for the attribute + hsize_t attrDims[1] = {value.size()}; + H5::DataSpace attribute_space(1, attrDims); + + H5::DataType type; + + // Create a int datatype + if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_INT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_ULONG; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_FLOAT; + } + else if constexpr (std::is_same_v) { + type = H5::PredType::NATIVE_DOUBLE; + } + else if constexpr (std::is_same_v) { + type = H5::StrType(0, H5T_VARIABLE); + } + else { + assert(0); + } + + // Create the attribute + H5::Attribute attribute = group.createAttribute(name, type, attribute_space); + + // Treat string separately + if constexpr (std::is_same_v) { + std::vector strVec; + for (auto & v : value) strVec.push_back(v.c_str()); + attribute.write(type, strVec.data()); + } else + attribute.write(type, value.data()); + }; + + +void OutHDF5::RunPSP(const std::string& path) +{ + int nOK = 0; + + // Begin HDF5 file writing + // + try { + // Create a file + H5::H5File file(path.c_str(), H5F_ACC_EXCL); + + try { + + // Create a new group for Header + // + H5::Group header = file.createGroup("Header"); + + int dp = real4 ? 0 : 1; + writeScalar(header, "Flag_DoublePrecision", dp); + + double hubble = 1, zero = 0; + + writeScalar(header, "HubbleParam", hubble); + writeScalar(header, "Omega0", zero ); + writeScalar(header, "OmegaBaryon", zero ); + writeScalar(header, "OmegaLambda", zero ); + writeScalar(header, "Redshift", zero ); + + if (masses.size()==0) checkParticleMasses(); + writeVector(header, "MassTable", masses); + + writeScalar(header, "NumFilesPerSnapshot", numprocs); + + std::vector nums(masses.size()); + { + int n=0; + for (auto c : comp->components) nums[n++] = c->Number(); + } + writeVector(header, "NumPart_ThisFile", nums); + { + int n=0; + for (auto c : comp->components) nums[n++] = c->CurTotal(); + } + writeVector(header, "NumPart_Total", nums); + + writeScalar(header, "Time", tnow); + + // Create a new group for Config + // + H5::Group config = file.createGroup("Config"); + + int style = 1; + writeScalar(config, "PSPstyle", style); + + int ntypes = comp->components.size(); + writeScalar(config, "NTYPES", ntypes); + + writeScalar(config, "DOUBLEPRECISION", dp); + + std::vector Niattrib, Ndattrib; + for (auto & c : comp->components) { + Niattrib.push_back(c->niattrib); + Ndattrib.push_back(c->ndattrib); + } + writeVector(config, "Niattrib", Niattrib); + writeVector(config, "Ndattrib", Ndattrib); + + // Create a new group for Parameters + // + H5::Group params = file.createGroup("Parameters"); + + std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); + writeScalar(params, "Git_commit", gcommit); + writeScalar(params, "Git_branch", gbranch); + writeScalar(params, "Compile_data", gdate ); + + std::vector names, forces, configs; + for (auto c : comp->components) { + names.push_back(c->name); + forces.push_back(c->id); + YAML::Emitter out; + out << c->fconf; // where node is your YAML::Node + configs.push_back(out.c_str()); + } + + writeVector(params, "ComponentNames", names); + writeVector(params, "ForceMethods", forces); + writeVector(params, "ForceConfigurations", configs); + + } catch (H5::Exception& error) { + throw std::runtime_error(std::string("OutHDF5: error writing HDF5 file ") + error.getDetailMsg()); + } + + int count = 0; + for (auto c : comp->components) { + +#ifdef HAVE_LIBCUDA + if (use_cuda) { + if (c->force->cudaAware() and not comp->fetched[c]) { + comp->fetched[c] = true; + c->CudaToParticles(); + } + } +#endif + + std::ostringstream sout; + sout << "PartType" << count++; + + H5::Group group = file.createGroup(sout.str()); + + if (real4) + c->write_H5(group); + else + c->write_H5(group); + } + } catch (H5::Exception& error) { + throw std::runtime_error(std::string("OutHDF5: error writing HDF5 file ") + error.getDetailMsg()); } } From ba499d9f5cf30f25cfc7175a199f33e80c655bf3 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 31 May 2025 13:44:21 -0400 Subject: [PATCH 055/106] Some more comments only --- exputil/ParticleReader.cc | 11 +++++++++-- src/Component.cc | 27 +++++++++++++++++---------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 15e55c1fe..109366295 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -857,6 +857,8 @@ namespace PR { T mass; T pos[3]; T vel[3]; + T pot; + T potext; hvl_t iattrib; hvl_t dattrib; }; @@ -989,8 +991,10 @@ namespace PR { hsize_t dims3[1] = {3}; H5::ArrayType array3_type(type, 1, dims3); - compound_type.insertMember("pos", HOFFSET(H5Particle, pos), array3_type); - compound_type.insertMember("vel", HOFFSET(H5Particle, vel), array3_type); + compound_type.insertMember("pos", HOFFSET(H5Particle, pos ), array3_type); + compound_type.insertMember("vel", HOFFSET(H5Particle, vel ), array3_type); + compound_type.insertMember("pot", HOFFSET(H5Particle, pot ), type); + compound_type.insertMember("potext", HOFFSET(H5Particle, potext), type); compound_type.insertMember("iattrib", HOFFSET(H5Particle, iattrib), H5::VarLenType(H5::PredType::NATIVE_INT)); compound_type.insertMember("dattrib", HOFFSET(H5Particle, dattrib), H5::VarLenType(type)); @@ -1059,6 +1063,9 @@ namespace PR { P.vel[k] = h5part[n].vel[k]; } + P.pot = h5part[n].pot; + P.potext = h5part[n].potext; + // Unpack the variable length attribute arrays // if (Niattrib[curindx]>0) { diff --git a/src/Component.cc b/src/Component.cc index 59a810c34..eb6882e6e 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2293,7 +2293,7 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) auto dcplI = HighFive::DataSetCreateProps{}; auto dcplD = HighFive::DataSetCreateProps{}; - if (H5compress) { + if (H5compress or H5chunk) { int chunk = H5chunk; // Sanity @@ -2349,7 +2349,8 @@ template void Component::write_HDF5 (HighFive::Group& group, bool masses, bool IDs); -// Helper structure +// Helper structure for reading and writing the HDF5 compound data +// type. Used by Component and ParticleReader. template struct H5Particle { @@ -2357,12 +2358,14 @@ struct H5Particle T mass; T pos[3]; T vel[3]; - hvl_t iattrib; - hvl_t dattrib; + T pot; + T potext; + hvl_t iattrib; // HDF5 variable length + hvl_t dattrib; // HDF5 variable length }; - -//! Write HDF5 phase-space structure for this nodes particles only. +// Write HDF5 phase-space structure for this nodes particles only in +// PSP style template void Component::write_H5(H5::Group& group) { @@ -2385,8 +2388,10 @@ void Component::write_H5(H5::Group& group) hsize_t dims3[1] = {3}; H5::ArrayType array3_type(type, 1, dims3); - compound_type.insertMember("pos", HOFFSET(H5Particle, pos), array3_type); - compound_type.insertMember("vel", HOFFSET(H5Particle, vel), array3_type); + compound_type.insertMember("pos", HOFFSET(H5Particle, pos ), array3_type); + compound_type.insertMember("vel", HOFFSET(H5Particle, vel ), array3_type); + compound_type.insertMember("pot", HOFFSET(H5Particle, pot ), type); + compound_type.insertMember("potext", HOFFSET(H5Particle, potext), type); compound_type.insertMember("iattrib", HOFFSET(H5Particle, iattrib), H5::VarLenType(H5::PredType::NATIVE_INT)); compound_type.insertMember("dattrib", HOFFSET(H5Particle, dattrib), H5::VarLenType(type)); @@ -2407,18 +2412,20 @@ void Component::write_H5(H5::Group& group) h5_particles[n].pos[k] = P->pos[k]; h5_particles[n].vel[k] = P->vel[k]; } + h5_particles[n].pot = P->pot; + h5_particles[n].potext = P->potext; h5_particles[n].iattrib.len = P->iattrib.size(); h5_particles[n].dattrib.len = P->dattrib.size(); h5_particles[n].iattrib.p = P->iattrib.data(); h5_particles[n].dattrib.p = P->dattrib.data(); } - // Set properties, if requested + // Set properties, if requested for chunking and compression H5::DSetCreatPropList dcpl; // This could be generalized by registering a user filter, like // blosc. Right now, we're using the default (which is gzip) - if (H5compress) { + if (H5compress or H5chunk) { int chunk = H5chunk; // Sanity From d762c1de604b927b914531c314f9e3026efd0dea Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 31 May 2025 14:16:41 -0400 Subject: [PATCH 056/106] Try to keep Clang happy... --- src/Component.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Component.cc b/src/Component.cc index eb6882e6e..033506cd2 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2434,7 +2434,7 @@ void Component::write_H5(H5::Group& group) } // Set chunking - hsize_t chunk_dims[1] = {chunk}; + hsize_t chunk_dims[1] = {static_cast(chunk)}; dcpl.setChunk(1, chunk_dims); // Set compression level From 3e1ec56722b217dacec49107480287ef60f1a5a4 Mon Sep 17 00:00:00 2001 From: mdw Date: Sat, 31 May 2025 15:50:39 -0400 Subject: [PATCH 057/106] Clean up H5 property settings --- src/Component.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Component.cc b/src/Component.cc index 033506cd2..f6200c9ed 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2426,22 +2426,22 @@ void Component::write_H5(H5::Group& group) // This could be generalized by registering a user filter, like // blosc. Right now, we're using the default (which is gzip) if (H5compress or H5chunk) { - int chunk = H5chunk; - - // Sanity - if (H5chunk >= nbodies) { - chunk = nbodies/8; - } - // Set chunking - hsize_t chunk_dims[1] = {static_cast(chunk)}; - dcpl.setChunk(1, chunk_dims); + if (H5chunk) { + // Sanity + int chunk = H5chunk; + if (H5chunk >= nbodies) { + chunk = nbodies/8; + } + hsize_t chunk_dims[1] = {static_cast(chunk)}; + dcpl.setChunk(1, chunk_dims); + } // Set compression level - dcpl.setDeflate(H5compress); + if (H5compress) dcpl.setDeflate(H5compress); // Enable shuffle filter - dcpl.setShuffle(); + if (H5shuffle) dcpl.setShuffle(); } // Create dataspace From aa43836fece4821cbe1953437ffb4823c90bf78d Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 31 May 2025 23:29:17 -0400 Subject: [PATCH 058/106] Preliminary untested restart from PSPhdf5 --- src/Component.H | 4 + src/Component.cc | 174 +++++++++++++++++++++++++++++++++ src/ComponentContainer.cc | 197 ++++++++++++++++++++++++++------------ 3 files changed, 313 insertions(+), 62 deletions(-) diff --git a/src/Component.H b/src/Component.H index 80aa8f77b..8be95eaa0 100644 --- a/src/Component.H +++ b/src/Component.H @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -548,6 +549,9 @@ public: //! Initialize from file Component(YAML::Node&, istream *in, bool SPL); + //! Initialize from HDF5 + Component(YAML::Node&, PR::PSPhdf5& reader); + //! Destructor ~Component(); diff --git a/src/Component.cc b/src/Component.cc index f6200c9ed..f810f4c7c 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -779,6 +779,180 @@ Component::Component(YAML::Node& CONF, istream *in, bool SPL) : conf(CONF) pbuf.resize(PFbufsz); } +Component::Component(YAML::Node& CONF, PR::PSPhdf5& reader) : conf(CONF) +{ + // Make a copy + conf = CONF; + + try { + name = conf["name"].as(); + } + catch (YAML::Exception & error) { + if (myid==0) std::cout << __FILE__ << ": " << __LINE__ << std::endl + << "Error parsing component 'name': " + << error.what() << std::endl + << std::string(60, '-') << std::endl + << "Config node" << std::endl + << std::string(60, '-') << std::endl + << conf << std::endl + << std::string(60, '-') << std::endl; + + throw std::runtime_error("Component: error parsing component "); + } + + try { + cconf = conf["parameters"]; + } + catch (YAML::Exception & error) { + if (myid==0) std::cout << "Error parsing 'parameters' for Component <" + << name << ">: " + << error.what() << std::endl + << std::string(60, '-') << std::endl + << "Config node" << std::endl + << std::string(60, '-') << std::endl + << conf + << std::string(60, '-') << std::endl; + + throw std::runtime_error("Component: error parsing "); + } + + pfile = conf["bodyfile"].as(); + + YAML::Node cforce; + try { + cforce = conf["force"]; + } + catch (YAML::Exception & error) { + if (myid==0) std::cout << "Error parsing 'force' for Component <" + << name << ">: " + << error.what() << std::endl + << std::string(60, '-') << std::endl + << "Config node" << std::endl + << std::string(60, '-') << std::endl + << conf << std::endl + << std::string(60, '-') << std::endl; + + throw std::runtime_error("Component: error parsing "); + } + + id = cforce["id"].as(); + + try { + fconf = cforce["parameters"]; + } + catch (YAML::Exception & error) { + if (myid==0) std::cout << "Error parsing force 'parameters' for Component <" + << name << ">: " + << error.what() << std::endl + << std::string(60, '-') << std::endl + << "Config node" << std::endl + << std::string(60, '-') << std::endl + << cforce << std::endl + << std::string(60, '-') << std::endl; + + throw std::runtime_error("Component: error parsing force "); + } + + // Defaults + // + EJ = 0; + nEJkeep = 100; + nEJwant = 500; + nEJaccel = 0; + EJkinE = true; + EJext = false; + EJdiag = false; + EJdryrun = false; + EJx0 = 0.0; + EJy0 = 0.0; + EJz0 = 0.0; + EJu0 = 0.0; + EJv0 = 0.0; + EJw0 = 0.0; + EJdT = 0.0; + EJlinear = false; + EJdamp = 1.0; + + binary = true; + + adiabatic = false; + ton = -1.0e20; + toff = 1.0e20; + twid = 0.1; + + rtrunc = 1.0e20; + rcom = 1.0e20; + consp = false; + tidal = -1; + + com_system = false; + com_log = false; + com_restart = 0; + c0 = NULL; // Component for centering (null by + // default) +#if HAVE_LIBCUDA==1 + bunchSize = 100000; +#endif + + timers = false; + + force = 0; // Null out pointers + orient = 0; + + com = 0; + cov = 0; + coa = 0; + center = 0; + angmom = 0; + ps = 0; + + com0 = 0; + cov0 = 0; + acc0 = 0; + + indexing = true; + aindex = false; + umagic = true; + + keyPos = -1; + nlevel = -1; + top_seq = 0; + + pBufSiz = 100000; + blocking = false; + buffered = true; // Use buffered writes for POSIX binary + noswitch = false; // Allow multistep switching at master step only + dtreset = true; // Select level from criteria over last step + freezeLev = false; // Only compute new levels on first step + + configure(); + + find_ctr_component(); + + initialize_cuda(); + + particles.clear(); + + // Read bodies + // + for (auto p=reader.firstParticle(); p!=0; p=reader.nextParticle()) { + particles[p->indx] = std::make_shared(*p); + } + + mdt_ctr = std::vector< std::vector > (multistep+1); + for (unsigned n=0; n<=multistep; n++) mdt_ctr[n] = std::vector(mdtDim, 0); + + angmom_lev = std::vector(3*(multistep+1), 0); + com_lev = std::vector(3*(multistep+1), 0); + cov_lev = std::vector(3*(multistep+1), 0); + coa_lev = std::vector(3*(multistep+1), 0); + com_mas = std::vector( multistep+1, 0); + + reset_level_lists(); + + pbuf.resize(PFbufsz); +} + void Component::configure(void) { // Check for unmatched keys diff --git a/src/ComponentContainer.cc b/src/ComponentContainer.cc index f399a59d0..aaba0a0be 100644 --- a/src/ComponentContainer.cc +++ b/src/ComponentContainer.cc @@ -4,12 +4,14 @@ #include +#include #include #include #include #include #include +#include #include #ifdef USE_GPTL @@ -46,27 +48,50 @@ void ComponentContainer::initialize(void) read_rates(); // Read initial processor rates - bool SPL = false; // Indicate whether file has SPL prefix - unsigned char ir = 0; - unsigned char is = 0; + bool SPL = false; // Indicates whether file has the SPL prefix + bool HDF5 = false; // Indicates an HDF5 restart directory + + unsigned short ir = 0; + unsigned short is = 0; + unsigned short ih = 0; // Look for a restart file if (myid==0) { - string resfile = outdir + infile; - ifstream in(resfile.c_str()); - if (in) { - if (ignore_info) - cerr << "---- ComponentContainer successfully opened <" - << resfile << ">, assuming a new run using a previous phase space as initial conditions" << endl; - else - cerr << "---- ComponentContainer successfully opened <" - << resfile << ">, assuming a restart" << endl; - ir = 1; + std::string resfile = outdir + infile; + + std::filesystem::path dir_path = resfile; + // If restart path is a directory, + // assume HDF5 + if (std::filesystem::is_directory(dir_path)) { + HDF5 = true; + try { + for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { + if (std::filesystem::is_regular_file(entry)) { + ir++; + auto filename = entry.path().filename().string(); + if (H5::H5File::isHdf5(filename.c_str())) ih++; + } + } + } catch (const std::filesystem::filesystem_error& e) { + std::cerr << "Component::initialize error: " << e.what() << std::endl; + } + } else { - cerr << "---- ComponentContainer could not open <" - << resfile << ">, assuming a new run" << endl; - ir = 0; + std::ifstream in(resfile.c_str()); + if (in) { + if (ignore_info) + cerr << "---- ComponentContainer successfully opened <" + << resfile << ">, assuming a new run using a previous phase space as initial conditions" << endl; + else + cerr << "---- ComponentContainer successfully opened <" + << resfile << ">, assuming a restart" << endl; + ir = 1; + } else { + cerr << "---- ComponentContainer could not open <" + << resfile << ">, assuming a new run" << endl; + ir = 0; + } + if (infile.find("SPL") != std::string::npos) is = 1; } - if (infile.find("SPL") != std::string::npos) is = 1; } MPI_Bcast(&ir, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); @@ -77,68 +102,116 @@ void ComponentContainer::initialize(void) if (restart) { - struct MasterHeader master; - ifstream in; + if (HDF5) { + if (ir != ih) + throw std::runtime_error("ComponentContainer::initialize HDF5 restart file count mismatch"); + + std::filesystem::path dir_path = outdir + infile; + std::vector files; + try { + for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { + if (std::filesystem::is_regular_file(entry)) files.push_back(entry.path().filename().string()); + } + } catch (const std::filesystem::filesystem_error& e) { + std::cerr << "ComponentContainer::initialize HDF5 file listing error: " << e.what() << std::endl; + } - // Open file - if (myid==0) { + PR::PSPhdf5 reader(files, true); + + auto types = reader.GetTypes(); - string resfile = outdir + infile; - in.open(resfile); - if (in.fail()) { - throw FileOpenError(resfile, __FILE__, __LINE__); + if (not ignore_info) tnow = reader.CurrentTime(); + + + if (myid==0) { + if (ignore_info) { + cout << "Found: " + << " Ntot=" << reader.CurrentNumber() + << " Ncomp=" << types.size() << std::endl; + + } else { + cout << "Recovering from: " + << " Tnow=" << tnow + << " Ntot=" << reader.CurrentNumber() + << " Ncomp=" << types.size() << endl; + } } - in.read((char *)&master, sizeof(MasterHeader)); - if (in.fail()) { - std::ostringstream sout; - sout << "ComponentContainer::initialize: " - << "could not read master header from <" - << resfile << ">"; - throw GenericError(sout.str(), __FILE__, __LINE__); + YAML::Node comp = parse["Components"]; + + // Will use ParticleReader to load particles without the usual + // ParticleFerry + if (comp.IsSequence()) { + for (int i=0; i"; + throw GenericError(sout.str(), __FILE__, __LINE__); + } + + if (ignore_info) { + cout << "Found: " + << " Ntot=" << master.ntot + << " Ncomp=" << master.ncomp << endl; + + } else { + cout << "Recovering from: " + << " Tnow=" << master.time + << " Ntot=" << master.ntot + << " Ncomp=" << master.ncomp << endl; - tnow = master.time; - } + tnow = master.time; + } - ntot = master.ntot; - ncomp = master.ncomp; - } + ntot = master.ntot; + ncomp = master.ncomp; + } - MPI_Bcast(&tnow, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); + MPI_Bcast(&tnow, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); - MPI_Bcast(&ntot, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Bcast(&ntot, 1, MPI_INT, 0, MPI_COMM_WORLD); - MPI_Bcast(&ncomp, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Bcast(&ncomp, 1, MPI_INT, 0, MPI_COMM_WORLD); - YAML::Node comp = parse["Components"]; + YAML::Node comp = parse["Components"]; - if (comp.IsSequence()) { - for (int i=0; i: " << e.what() << std::endl; + try { + in.close(); + } + catch (const ifstream::failure& e) { + std::cout << "ComponentContainer: exception closing file <" + << outdir + infile << ">: " << e.what() << std::endl; + } } } else { From f836d475bf2796e43921e3b12c3400d9212028d3 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 1 Jun 2025 11:48:33 -0400 Subject: [PATCH 059/106] Preliminary, partially tested implementation of restart from PSPhdf5 --- exputil/ParticleReader.cc | 6 ++++++ src/Component.cc | 29 +++++++++++++++++++++++++++++ src/ComponentContainer.cc | 25 ++++++++++++++++--------- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 109366295..0496b8a1f 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -779,11 +779,14 @@ namespace PR { HighFive::Group part = h5file.getGroup(sout.str()); Eigen::Matrix pos, vel; + std::vector idx; std::vector mas; Eigen::Matrix iattrib; Eigen::Matrix rattrib; + part.getDataSet("ParticleIDs" ).read(idx); + if (mass[curindx] == 0) part.getDataSet("Masses").read(mas); @@ -807,6 +810,8 @@ namespace PR { for (int n=0; n 0) P.mass = mass[curindx]; else P.mass = mas[n]; @@ -1055,6 +1060,7 @@ namespace PR { // Assign the standard fields // + P.indx = h5part[n].id; P.mass = h5part[n].mass; P.level = 0; diff --git a/src/Component.cc b/src/Component.cc index f810f4c7c..446b8469d 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -935,10 +935,39 @@ Component::Component(YAML::Node& CONF, PR::PSPhdf5& reader) : conf(CONF) // Read bodies // + double rmax1=0.0; + niattrib=-1; for (auto p=reader.firstParticle(); p!=0; p=reader.nextParticle()) { + // Assign attribute sizes + if (niattrib<0) { + niattrib = p->iattrib.size(); + ndattrib = p->dattrib.size(); + } + + // Make the particle particles[p->indx] = std::make_shared(*p); + + // Get limits + double r2 = 0.0; + for (int k=0; k<3; k++) r2 += p->pos[k]*p->pos[k]; + rmax1 = std::max(r2, rmax1); + top_seq = std::max(top_seq, p->indx); } + // Default: set to max radius + rmax = sqrt(fabs(rmax1)); + MPI_Bcast(&rmax, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); + + // Send top_seq to all nodes + MPI_Bcast(&top_seq, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD); + + initialize(); + + // Initialize the particle ferry + // instance with dynamic attribute + // sizes + if (not pf) pf = ParticleFerryPtr(new ParticleFerry(niattrib, ndattrib)); + mdt_ctr = std::vector< std::vector > (multistep+1); for (unsigned n=0; n<=multistep; n++) mdt_ctr[n] = std::vector(mdtDim, 0); diff --git a/src/ComponentContainer.cc b/src/ComponentContainer.cc index aaba0a0be..787ab9c16 100644 --- a/src/ComponentContainer.cc +++ b/src/ComponentContainer.cc @@ -49,7 +49,7 @@ void ComponentContainer::initialize(void) bool SPL = false; // Indicates whether file has the SPL prefix - bool HDF5 = false; // Indicates an HDF5 restart directory + bool HDF5 = false; // Indicates whether file is an HDF5 file unsigned short ir = 0; unsigned short is = 0; @@ -67,8 +67,7 @@ void ComponentContainer::initialize(void) for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { if (std::filesystem::is_regular_file(entry)) { ir++; - auto filename = entry.path().filename().string(); - if (H5::H5File::isHdf5(filename.c_str())) ih++; + if (H5::H5File::isHdf5(entry.path().string())) ih++; } } } catch (const std::filesystem::filesystem_error& e) { @@ -94,8 +93,10 @@ void ComponentContainer::initialize(void) } } - MPI_Bcast(&ir, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); - MPI_Bcast(&is, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(&ir, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(&is, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(&ih, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); + MPI_Bcast(&HDF5, 1, MPI_CXX_BOOL, 0, MPI_COMM_WORLD); restart = ir ? true : false; SPL = is ? true : false; @@ -103,15 +104,19 @@ void ComponentContainer::initialize(void) if (restart) { if (HDF5) { - if (ir != ih) - throw std::runtime_error("ComponentContainer::initialize HDF5 restart file count mismatch"); + + // Sanity check: must have at least one HDF5 file in the + // directory + if (ih<1) + throw std::runtime_error("ComponentContainer::initialize HDF5 restart directory found but no HDF5 files found"); std::filesystem::path dir_path = outdir + infile; std::vector files; try { for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { - if (std::filesystem::is_regular_file(entry)) files.push_back(entry.path().filename().string()); - } + auto file = entry.path().string(); + if (H5::H5File::isHdf5(file)) files.push_back(file); + } } catch (const std::filesystem::filesystem_error& e) { std::cerr << "ComponentContainer::initialize HDF5 file listing error: " << e.what() << std::endl; } @@ -119,6 +124,7 @@ void ComponentContainer::initialize(void) PR::PSPhdf5 reader(files, true); auto types = reader.GetTypes(); + ncomp = types.size(); if (not ignore_info) tnow = reader.CurrentTime(); @@ -144,6 +150,7 @@ void ComponentContainer::initialize(void) if (comp.IsSequence()) { for (int i=0; i Date: Sun, 1 Jun 2025 13:17:40 -0400 Subject: [PATCH 060/106] Version bump --- CMakeLists.txt | 2 +- doc/exp.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a855b74f..5fe8eb30e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.25) # Needed for CUDA, MPI, and CTest features project( EXP - VERSION "7.9.0" + VERSION "7.9.1" HOMEPAGE_URL https://github.com/EXP-code/EXP LANGUAGES C CXX Fortran) diff --git a/doc/exp.cfg b/doc/exp.cfg index eb9d53c5c..0d3fc92e0 100644 --- a/doc/exp.cfg +++ b/doc/exp.cfg @@ -48,7 +48,7 @@ PROJECT_NAME = EXP # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 7.8.1 +PROJECT_NUMBER = 7.9.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From b0d5a6b794ff49a3ecbc312a707cdf29b0f8c24c Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 1 Jun 2025 13:18:01 -0400 Subject: [PATCH 061/106] Added a checkpoint-file writing mode --- src/OutHDF5.H | 4 +++- src/OutHDF5.cc | 59 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index d3d256141..b1b0db8c2 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -14,6 +14,7 @@ @param threads number of threads for binary writes @param gadget set to true for Gadget-4-type HDF5 @param noids set to true turns off particle id writing + @param checkpt sets checkpoint mode: write a real8 snapshot with a unique filename */ class OutHDF5 : public Output @@ -22,7 +23,7 @@ class OutHDF5 : public Output private: std::string filename; - bool real4=true, real8=false, ids=true, gadget4=false, timer; + bool real4=true, real8=false, ids=true, gadget4=false, chkpt=false, timer; int nbeg, threads; void initialize(void); std::vector masses; @@ -57,6 +58,7 @@ public: \param threads is the thread count for binary writes \param noids set to true turns off particle id writing \param gadget set to true for Gadget-4-type HDF5 + \param checkpt set to true for checkpoint mode */ void Run(int nstep, int mstep, bool last); diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 7492a8521..48630d063 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -5,8 +5,13 @@ #include #include -// HighFive API for HDF5 +// For unlink +#include + +// H5 API #include + +// HighFive API for HDF5 #include #include @@ -27,6 +32,7 @@ OutHDF5::valid_keys = { "timer", "threads," "gadget", + "checkpt", "H5compress", "H5chunk" }; @@ -87,6 +93,13 @@ void OutHDF5::initialize() real4 = not real8; } + // This will override user setting of real4... + if (Output::conf["checkpt"]) { + chkpt = true; + real8 = true; + real4 = false; + } + if (Output::conf["noids"]) { ids = not Output::conf["noids"].as(); } @@ -184,11 +197,47 @@ void OutHDF5::Run(int n, int mstep, bool last) std::ofstream out; std::ostringstream fname; - // Output name prefix - fname << filename << "_" << setw(5) << setfill('0') << nbeg++; - if (numprocs>1) fname << "." << myid+1; + // Checkpoint mode + if (chkpt) { + // Create checkpoint filename + fname << "checkpoint_" << runtag; + if (numprocs>1) fname << "." << myid+1; + + // Create backup filename + std::string currfile = outdir + fname.str(); + std::string backfile = currfile + ".bak"; + + // Remove old backup file + if (unlink(backfile.c_str())) { + if (VERBOSE>5) perror("OutHDF5::Run()"); + std::cout << "OutHDF5::Run(): error unlinking old backup file <" + << backfile << ">, it may not exist" << std::endl; + } else { + if (VERBOSE>5) { + std::cout << "OutHDF5::Run(): successfully unlinked <" + << backfile << ">" << std::endl; + } + } + + // Rename current file to backup file + if (rename(currfile.c_str(), backfile.c_str())) { + if (VERBOSE>5) perror("OutHDF5::Run()"); + std::cout << "OutHDF5: renaming backup file <" + << backfile << ">, it may not exist" << std::endl; + } else { + if (VERBOSE>5) { + std::cout << "OutHDF5::Run(): successfully renamed <" + << currfile << "> to <" << backfile << ">" << std::endl; + } + } + } + // Standard snapshot mode + else { + fname << filename << "_" << setw(5) << setfill('0') << nbeg++; + if (numprocs>1) fname << "." << myid+1; + } - // Master file name + // Full path std::string path = outdir + fname.str(); if (gadget4) RunGadget4(path); From 21a2f5a1eee500e0e5c2e31f67b11c061cd189b7 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 1 Jun 2025 13:54:29 -0400 Subject: [PATCH 062/106] Check found HDF5 files with expected HDF5 files as a sanity check --- exputil/ParticleReader.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 0496b8a1f..44efe8e4f 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -646,6 +646,11 @@ namespace PR { _verbose = verbose; getInfo(); + + // Sanity check + if (nfiles != files.size()) + throw GenericError("PSPhdf5: number of files does not match number expected for this snapshot", __FILE__, __LINE__, 1042, true); + curfile = _files.begin(); if (not nextFile()) { @@ -682,6 +687,10 @@ namespace PR { // header.getAttribute("MassTable").read(mass); + // Number of files + // + header.getAttribute("NumFilesPerSnapshot").read(nfiles); + // Get component names // params.getAttribute("ComponentNames").read(comps); From 55f7d8ebcb3f8aefad423a5ca79621c9738b7e54 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 1 Jun 2025 13:54:52 -0400 Subject: [PATCH 063/106] Check found HDF5 files with expected HDF5 files as a sanity check --- include/ParticleReader.H | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/ParticleReader.H b/include/ParticleReader.H index a01970305..3c835dbf1 100644 --- a/include/ParticleReader.H +++ b/include/ParticleReader.H @@ -243,7 +243,7 @@ namespace PR std::vector npart, nptot; unsigned long totalCount; bool real4, gadget4; - int ntypes; + int ntypes, nfiles; //@} //! Current file @@ -306,6 +306,9 @@ namespace PR //! Get the next particle virtual const Particle* nextParticle(); + //! Get file cound per snapshot + virtual int NumFiles() { return nfiles; } + }; class PSPstanza From 287e01833085ccdc1274338a0921d0103154e1b6 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 1 Jun 2025 20:47:38 -0400 Subject: [PATCH 064/106] Set default HDF5 write mode to 'trunc' (overwrite) with an option for 'excl' (no overwrite) selective with the {preserve: bool} option --- src/OutHDF5.H | 6 +++++- src/OutHDF5.cc | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index b1b0db8c2..f45c833cd 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -15,6 +15,7 @@ @param gadget set to true for Gadget-4-type HDF5 @param noids set to true turns off particle id writing @param checkpt sets checkpoint mode: write a real8 snapshot with a unique filename + @param preserve prevents overwrite of snapshot files */ class OutHDF5 : public Output @@ -23,7 +24,9 @@ class OutHDF5 : public Output private: std::string filename; - bool real4=true, real8=false, ids=true, gadget4=false, chkpt=false, timer; + bool real4=true, real8=false, ids=true, gadget4=false, chkpt=false; + bool overwrite=false, timer; + unsigned write_flags; int nbeg, threads; void initialize(void); std::vector masses; @@ -59,6 +62,7 @@ public: \param noids set to true turns off particle id writing \param gadget set to true for Gadget-4-type HDF5 \param checkpt set to true for checkpoint mode + \param preserve set to true prevents overwrite of snapshot files */ void Run(int nstep, int mstep, bool last); diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 48630d063..72fc7c1cc 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -33,6 +33,7 @@ OutHDF5::valid_keys = { "threads," "gadget", "checkpt", + "preserve", "H5compress", "H5chunk" }; @@ -100,6 +101,12 @@ void OutHDF5::initialize() real4 = false; } + write_flags = H5F_ACC_TRUNC; + if (Output::conf["preserve"]) { + bool overwrite = Output::conf["preserve"].as(); + write_flags = preserve ? H5F_ACC_EXCL : H5F_ACC_TRUNC; + } + if (Output::conf["noids"]) { ids = not Output::conf["noids"].as(); } @@ -494,7 +501,7 @@ void OutHDF5::RunPSP(const std::string& path) // try { // Create a file - H5::H5File file(path.c_str(), H5F_ACC_EXCL); + H5::H5File file(path.c_str(), write_flags); try { From db156c4f71b5ea7ae9b7dce08d529103bd2cae28 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 1 Jun 2025 21:05:12 -0400 Subject: [PATCH 065/106] stdout log changes --- src/ComponentContainer.cc | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/ComponentContainer.cc b/src/ComponentContainer.cc index 787ab9c16..56dbcc060 100644 --- a/src/ComponentContainer.cc +++ b/src/ComponentContainer.cc @@ -78,15 +78,15 @@ void ComponentContainer::initialize(void) std::ifstream in(resfile.c_str()); if (in) { if (ignore_info) - cerr << "---- ComponentContainer successfully opened <" - << resfile << ">, assuming a new run using a previous phase space as initial conditions" << endl; + std::cerr << "---- ComponentContainer successfully opened <" + << resfile << ">, assuming a new run using a previous phase space as initial conditions" << std::endl; else - cerr << "---- ComponentContainer successfully opened <" - << resfile << ">, assuming a restart" << endl; + std::cerr << "---- ComponentContainer successfully opened <" + << resfile << ">, assuming a restart" << std::endl; ir = 1; } else { - cerr << "---- ComponentContainer could not open <" - << resfile << ">, assuming a new run" << endl; + std::cerr << "---- ComponentContainer could not open <" + << resfile << ">, assuming a new run" << std::endl; ir = 0; } if (infile.find("SPL") != std::string::npos) is = 1; @@ -131,15 +131,15 @@ void ComponentContainer::initialize(void) if (myid==0) { if (ignore_info) { - cout << "Found: " + cout << "---- ComponentContainer found: " << " Ntot=" << reader.CurrentNumber() << " Ncomp=" << types.size() << std::endl; } else { - cout << "Recovering from: " + cout << "---- ComponentContainer recovering from: " << " Tnow=" << tnow << " Ntot=" << reader.CurrentNumber() - << " Ncomp=" << types.size() << endl; + << " Ncomp=" << types.size() << std::endl; } } @@ -178,15 +178,15 @@ void ComponentContainer::initialize(void) } if (ignore_info) { - cout << "Found: " + cout << "---- ComponentContainer found: " << " Ntot=" << master.ntot - << " Ncomp=" << master.ncomp << endl; + << " Ncomp=" << master.ncomp << std::endl; } else { - cout << "Recovering from: " + cout << "---- ComponentContainer recovering from: " << " Tnow=" << master.time << " Ntot=" << master.ntot - << " Ncomp=" << master.ncomp << endl; + << " Ncomp=" << master.ncomp << std::endl; tnow = master.time; } From 816c96cfca6e45dfc13777cd9ff646cb0ac51420 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sun, 1 Jun 2025 21:05:42 -0400 Subject: [PATCH 066/106] Make HDF5 overwrite the default --- src/OutHDF5.H | 4 +--- src/OutHDF5.cc | 8 +------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index f45c833cd..c70d6f0ba 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -11,7 +11,6 @@ @param nint is the number of steps between dumps @param nbeg is suffix of the first phase space %dump @param timer set to true turns on wall-clock timer for PS output - @param threads number of threads for binary writes @param gadget set to true for Gadget-4-type HDF5 @param noids set to true turns off particle id writing @param checkpt sets checkpoint mode: write a real8 snapshot with a unique filename @@ -27,7 +26,7 @@ private: bool real4=true, real8=false, ids=true, gadget4=false, chkpt=false; bool overwrite=false, timer; unsigned write_flags; - int nbeg, threads; + int nbeg; void initialize(void); std::vector masses; std::vector multim; @@ -58,7 +57,6 @@ public: \param last should be true on final step to force phase space %dump indepentently of whether or not the frequency criterion is met \param timer set to true turns on wall-clock timer for PS output - \param threads is the thread count for binary writes \param noids set to true turns off particle id writing \param gadget set to true for Gadget-4-type HDF5 \param checkpt set to true for checkpoint mode diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 72fc7c1cc..2debd1653 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -30,7 +30,6 @@ OutHDF5::valid_keys = { "real4", "real8", "timer", - "threads," "gadget", "checkpt", "preserve", @@ -103,7 +102,7 @@ void OutHDF5::initialize() write_flags = H5F_ACC_TRUNC; if (Output::conf["preserve"]) { - bool overwrite = Output::conf["preserve"].as(); + bool preserve = Output::conf["preserve"].as(); write_flags = preserve ? H5F_ACC_EXCL : H5F_ACC_TRUNC; } @@ -116,11 +115,6 @@ void OutHDF5::initialize() else timer = false; - if (Output::conf["threads"]) - threads = Output::conf["threads"].as(); - else - threads = 0; - if (Output::conf["gadget"]) gadget4 = Output::conf["gadget"].as(); else From e54f5c87ec641e80e32cd4d7a64b512023361790 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 2 Jun 2025 13:58:20 -0400 Subject: [PATCH 067/106] Cache the entire EXP yaml config in the snapshot; controlled by the 'expconfig' boolean flag --- src/OutHDF5.H | 4 +++- src/OutHDF5.cc | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index c70d6f0ba..32b602d83 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -15,6 +15,7 @@ @param noids set to true turns off particle id writing @param checkpt sets checkpoint mode: write a real8 snapshot with a unique filename @param preserve prevents overwrite of snapshot files + @param expconfig set to true stashes the YAML configuration file in the HDF5 file */ class OutHDF5 : public Output @@ -24,7 +25,7 @@ private: std::string filename; bool real4=true, real8=false, ids=true, gadget4=false, chkpt=false; - bool overwrite=false, timer; + bool overwrite=false, expconfig=true, timer; unsigned write_flags; int nbeg; void initialize(void); @@ -61,6 +62,7 @@ public: \param gadget set to true for Gadget-4-type HDF5 \param checkpt set to true for checkpoint mode \param preserve set to true prevents overwrite of snapshot files + \param expconfig set to true stashes the YAML configuration file in the HDF5 file */ void Run(int nstep, int mstep, bool last); diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 2debd1653..a42ff8fb3 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -34,7 +34,8 @@ OutHDF5::valid_keys = { "checkpt", "preserve", "H5compress", - "H5chunk" + "H5chunk", + "expconfig" }; OutHDF5::OutHDF5(const YAML::Node& conf) : Output(conf) @@ -120,6 +121,9 @@ void OutHDF5::initialize() else gadget4 = false; + if (Output::conf["expconfig"]) + expconfg = Output::conf["expconfig"].as(); + // Default HDF5 compression is no compression. By default, // shuffle is on unless turned off manually. // @@ -358,6 +362,15 @@ void OutHDF5::RunGadget4(const std::string& path) params.createAttribute("ForceMethods", forces); params.createAttribute("ForceConfigurations", configs); + if (expconfig) { + // Save the entire EXP YAML configuration + YAML::Emitter cparse; + cparse << parse; + + std::string cyml(cparse.c_str()); + params.createAttribute("EXPConfiguration", cyml); + } + } catch (HighFive::Exception& err) { std::string msg("OutHDF5: error writing HDF5 file, "); throw std::runtime_error(msg + err.what()); @@ -575,6 +588,14 @@ void OutHDF5::RunPSP(const std::string& path) writeVector(params, "ForceMethods", forces); writeVector(params, "ForceConfigurations", configs); + if (expconfig) { + // Save the entire EXP YAML configuration + YAML::Emitter cparse; + cparse << parse; + + writeScalar(params, "EXPConfiguration", std::string(cparse.c_str())); + } + } catch (H5::Exception& error) { throw std::runtime_error(std::string("OutHDF5: error writing HDF5 file ") + error.getDetailMsg()); } From 0ff9ea4a9f623c4a921aea4a36d12133fcb68177 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 2 Jun 2025 16:38:16 -0400 Subject: [PATCH 068/106] Fix typo; consolidate metadata --- src/OutHDF5.cc | 263 +++++++++++++++++++++++++------------------------ 1 file changed, 136 insertions(+), 127 deletions(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index a42ff8fb3..a025139f8 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -122,7 +122,7 @@ void OutHDF5::initialize() gadget4 = false; if (Output::conf["expconfig"]) - expconfg = Output::conf["expconfig"].as(); + expconfig = Output::conf["expconfig"].as(); // Default HDF5 compression is no compression. By default, // shuffle is on unless turned off manually. @@ -281,94 +281,99 @@ void OutHDF5::RunGadget4(const std::string& path) HighFive::File::ReadWrite | HighFive::File::Create); try { - // Create a new group for Header // HighFive::Group header = file->createGroup("Header"); - int dp = 1; - header.createAttribute("Flag_DoublePrecision", dp); - - double hubble = 1, zero = 0; - header.createAttribute("HubbleParam", hubble); - - header.createAttribute("Omega0", zero); - - header.createAttribute("OmegaBaryon", zero); - - header.createAttribute("OmegaLambda", zero); - - header.createAttribute("Redshift", zero); if (masses.size()==0) checkParticleMasses(); header.createAttribute("MassTable", masses); - - header.createAttribute("NumFilesPerSnapshot", numprocs); - + std::vector nums(masses.size()); { int n=0; for (auto c : comp->components) nums[n++] = c->Number(); } header.createAttribute("NumPart_ThisFile", nums); - { - int n=0; - for (auto c : comp->components) nums[n++] = c->CurTotal(); - } - header.createAttribute("NumPart_Total", nums); - header.createAttribute("Time", tnow); - // Create a new group for Config + // Only attach metadata to the first file // - HighFive::Group config = file->createGroup("Config"); + if (myid==0) { - int style = 0; - config.createAttribute("PSPstyle", style); + int dp = 1; + header.createAttribute("Flag_DoublePrecision", dp); - int ntypes = comp->components.size(); - config.createAttribute("NTYPES", ntypes); - - config.createAttribute("DOUBLEPRECISION", dp); - - std::vector Niattrib, Ndattrib; - for (auto & c : comp->components) { - Niattrib.push_back(c->niattrib); - Ndattrib.push_back(c->ndattrib); - } + double hubble = 1, zero = 0; + header.createAttribute("HubbleParam", hubble); + + header.createAttribute("Omega0", zero); + + header.createAttribute("OmegaBaryon", zero); + + header.createAttribute("OmegaLambda", zero); + + header.createAttribute("Redshift", zero); + + header.createAttribute("NumFilesPerSnapshot", numprocs); + + { + int n=0; + for (auto c : comp->components) nums[n++] = c->CurTotal(); + } + header.createAttribute("NumPart_Total", nums); + + // Create a new group for Config + // + HighFive::Group config = file->createGroup("Config"); - config.createAttribute("Niattrib", Niattrib); + int style = 0; + config.createAttribute("PSPstyle", style); + + int ntypes = comp->components.size(); + config.createAttribute("NTYPES", ntypes); + + config.createAttribute("DOUBLEPRECISION", dp); - config.createAttribute("Ndattrib", Ndattrib); + std::vector Niattrib, Ndattrib; + for (auto & c : comp->components) { + Niattrib.push_back(c->niattrib); + Ndattrib.push_back(c->ndattrib); + } - // Create a new group for Parameters - // - HighFive::Group params = file->createGroup("Parameters"); - - std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); - params.createAttribute("Git_commit", gcommit); - params.createAttribute("Git_branch", gbranch); - params.createAttribute("Compile_date", gdate); - - std::vector names, forces, configs; - for (auto c : comp->components) { - names.push_back(c->name); - forces.push_back(c->id); - YAML::Emitter out; - out << c->fconf; // where node is your YAML::Node - configs.push_back(out.c_str()); - } + config.createAttribute("Niattrib", Niattrib); - params.createAttribute("ComponentNames", names); - params.createAttribute("ForceMethods", forces); - params.createAttribute("ForceConfigurations", configs); - - if (expconfig) { - // Save the entire EXP YAML configuration - YAML::Emitter cparse; - cparse << parse; + config.createAttribute("Ndattrib", Ndattrib); + + // Create a new group for Parameters + // + HighFive::Group params = file->createGroup("Parameters"); - std::string cyml(cparse.c_str()); - params.createAttribute("EXPConfiguration", cyml); + std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); + params.createAttribute("Git_commit", gcommit); + params.createAttribute("Git_branch", gbranch); + params.createAttribute("Compile_date", gdate); + + std::vector names, forces, configs; + for (auto c : comp->components) { + names.push_back(c->name); + forces.push_back(c->id); + YAML::Emitter out; + out << c->fconf; // where node is your YAML::Node + configs.push_back(out.c_str()); + } + + params.createAttribute("ComponentNames", names); + params.createAttribute("ForceMethods", forces); + params.createAttribute("ForceConfigurations", configs); + + if (expconfig) { + // Save the entire EXP YAML configuration + YAML::Emitter cparse; + cparse << parse; + + std::string cyml(cparse.c_str()); + params.createAttribute("EXPConfiguration", cyml); + } } } catch (HighFive::Exception& err) { @@ -516,86 +521,90 @@ void OutHDF5::RunPSP(const std::string& path) // H5::Group header = file.createGroup("Header"); - int dp = real4 ? 0 : 1; - writeScalar(header, "Flag_DoublePrecision", dp); - - double hubble = 1, zero = 0; - - writeScalar(header, "HubbleParam", hubble); - writeScalar(header, "Omega0", zero ); - writeScalar(header, "OmegaBaryon", zero ); - writeScalar(header, "OmegaLambda", zero ); - writeScalar(header, "Redshift", zero ); - if (masses.size()==0) checkParticleMasses(); writeVector(header, "MassTable", masses); - writeScalar(header, "NumFilesPerSnapshot", numprocs); - std::vector nums(masses.size()); { int n=0; for (auto c : comp->components) nums[n++] = c->Number(); } writeVector(header, "NumPart_ThisFile", nums); - { - int n=0; - for (auto c : comp->components) nums[n++] = c->CurTotal(); - } - writeVector(header, "NumPart_Total", nums); writeScalar(header, "Time", tnow); - - // Create a new group for Config + + // Only attach metadata to first process // - H5::Group config = file.createGroup("Config"); + if (myid==0) { + int dp = real4 ? 0 : 1; + writeScalar(header, "Flag_DoublePrecision", dp); - int style = 1; - writeScalar(config, "PSPstyle", style); + double hubble = 1, zero = 0; - int ntypes = comp->components.size(); - writeScalar(config, "NTYPES", ntypes); + writeScalar(header, "HubbleParam", hubble); + writeScalar(header, "Omega0", zero ); + writeScalar(header, "OmegaBaryon", zero ); + writeScalar(header, "OmegaLambda", zero ); + writeScalar(header, "Redshift", zero ); + + writeScalar(header, "NumFilesPerSnapshot", numprocs); + + { + int n=0; + for (auto c : comp->components) nums[n++] = c->CurTotal(); + } + writeVector(header, "NumPart_Total", nums); - writeScalar(config, "DOUBLEPRECISION", dp); + // Create a new group for Config + // + H5::Group config = file.createGroup("Config"); + + int style = 1; + writeScalar(config, "PSPstyle", style); + + int ntypes = comp->components.size(); + writeScalar(config, "NTYPES", ntypes); - std::vector Niattrib, Ndattrib; - for (auto & c : comp->components) { - Niattrib.push_back(c->niattrib); - Ndattrib.push_back(c->ndattrib); - } - writeVector(config, "Niattrib", Niattrib); - writeVector(config, "Ndattrib", Ndattrib); + writeScalar(config, "DOUBLEPRECISION", dp); - // Create a new group for Parameters - // - H5::Group params = file.createGroup("Parameters"); + std::vector Niattrib, Ndattrib; + for (auto & c : comp->components) { + Niattrib.push_back(c->niattrib); + Ndattrib.push_back(c->ndattrib); + } + writeVector(config, "Niattrib", Niattrib); + writeVector(config, "Ndattrib", Ndattrib); + + // Create a new group for Parameters + // + H5::Group params = file.createGroup("Parameters"); - std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); - writeScalar(params, "Git_commit", gcommit); - writeScalar(params, "Git_branch", gbranch); - writeScalar(params, "Compile_data", gdate ); - - std::vector names, forces, configs; - for (auto c : comp->components) { - names.push_back(c->name); - forces.push_back(c->id); - YAML::Emitter out; - out << c->fconf; // where node is your YAML::Node - configs.push_back(out.c_str()); - } - - writeVector(params, "ComponentNames", names); - writeVector(params, "ForceMethods", forces); - writeVector(params, "ForceConfigurations", configs); + std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); + writeScalar(params, "Git_commit", gcommit); + writeScalar(params, "Git_branch", gbranch); + writeScalar(params, "Compile_data", gdate ); + + std::vector names, forces, configs; + for (auto c : comp->components) { + names.push_back(c->name); + forces.push_back(c->id); + YAML::Emitter out; + out << c->fconf; // where node is your YAML::Node + configs.push_back(out.c_str()); + } + + writeVector(params, "ComponentNames", names); + writeVector(params, "ForceMethods", forces); + writeVector(params, "ForceConfigurations", configs); - if (expconfig) { - // Save the entire EXP YAML configuration - YAML::Emitter cparse; - cparse << parse; - - writeScalar(params, "EXPConfiguration", std::string(cparse.c_str())); + if (expconfig) { + // Save the entire EXP YAML configuration + YAML::Emitter cparse; + cparse << parse; + + writeScalar(params, "EXPConfiguration", std::string(cparse.c_str())); + } } - } catch (H5::Exception& error) { throw std::runtime_error(std::string("OutHDF5: error writing HDF5 file ") + error.getDetailMsg()); } From 53c2e8e0944e81d831e44bcc2858b60488083074 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 2 Jun 2025 16:38:35 -0400 Subject: [PATCH 069/106] Generalize diffpsp for PSPhdf5 --- utils/PhaseSpace/diffpsp.cc | 88 ++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 10 deletions(-) diff --git a/utils/PhaseSpace/diffpsp.cc b/utils/PhaseSpace/diffpsp.cc index e396d9997..736e79783 100755 --- a/utils/PhaseSpace/diffpsp.cc +++ b/utils/PhaseSpace/diffpsp.cc @@ -47,6 +47,8 @@ #include +#include + #include #include #include @@ -173,7 +175,7 @@ main(int argc, char **argv) ("jaco", "Compute phase-space Jacobian for DF computation") ("Emass", "Create energy bins approximately uniform in mass using potential from the mass model") ("actions", "Print output in action space rather than E-kappa space. The default is Energy-Kappa.") - ("F,filetype", "input file type (one of: PSPout, PSPspl, GadgetNative, GadgetHDF5)", + ("F,filetype", "input file type (one of: PSPout, PSPspl, PSPhdf5, GadgetNative, GadgetHDF5)", cxxopts::value(fileType)->default_value("PSPout")) ("I1min", "Minimum grid value for E (or I1 for actions)", cxxopts::value(I1min)) @@ -327,8 +329,30 @@ main(int argc, char **argv) // Check for existence of files in INFILE1 and INFILE2 lists // unsigned bad = 0; - for (auto file : INFILE1) { - if (not std::filesystem::exists(std::filesystem::path(file))) bad++; + std::string path1, path2; + if (fileType == "PSPhdf5") { + path1 = CURDIR + INFILE1[0]; + std::filesystem::path dir_path = path1; + if (std::filesystem::is_directory(dir_path)) { + INFILE1.clear(); + try { + for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { + if (std::filesystem::is_regular_file(entry)) { + std::string file = entry.path().string(); + if (H5::H5File::isHdf5(file)) INFILE1.push_back(file); + } + } + } catch (const std::filesystem::filesystem_error& e) { + std::cerr << "diffpsp error: " << e.what() << std::endl; + } + + if (INFILE1.size()==0) bad++; + } + } + else { + for (auto file : INFILE1) { + if (not std::filesystem::exists(std::filesystem::path(file))) bad++; + } } if (bad) { @@ -339,8 +363,29 @@ main(int argc, char **argv) exit(-1); } - for (auto file : INFILE2) { - if (not std::filesystem::exists(std::filesystem::path(file))) bad++; + if (fileType == "PSPhdf5") { + path2 = CURDIR + INFILE2[0]; + std::filesystem::path dir_path = path2; + if (std::filesystem::is_directory(dir_path)) { + INFILE2.clear(); + try { + for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { + if (std::filesystem::is_regular_file(entry)) { + std::string file = entry.path().string(); + if (H5::H5File::isHdf5(file)) INFILE2.push_back(file); + } + } + } catch (const std::filesystem::filesystem_error& e) { + std::cerr << "diffpsp error: " << e.what() << std::endl; + } + + if (INFILE2.size()==0) bad++; + } + } + else { + for (auto file : INFILE2) { + if (not std::filesystem::exists(std::filesystem::path(file))) bad++; + } } if (bad) { @@ -579,15 +624,27 @@ main(int argc, char **argv) // double initl_time, final_time; + // Number of paths + // + int npath1 = 1, npath2 = 1; + if (fileType != "PSPhdf5") { + npath1 = INFILE1.size(); + npath2 = INFILE1.size(); + } // Iterate through file list // - for (size_t n=0; nCurrentTime(); @@ -595,7 +652,10 @@ main(int argc, char **argv) if (myid==0) { std::cout << std::endl << std::string(40, '-') << std::endl; - std::cout << "File 1: " << INFILE1[n] << std::endl; + if (fileType == "PSPhdf5") + std::cout << "Path 1: " << path1 << std::endl; + else + std::cout << "File 1: " << CURDIR + INFILE1[n] << std::endl; std::cout << "Found dump at time: " << initl_time << std::endl; } } @@ -610,14 +670,22 @@ main(int argc, char **argv) } try { - psp2 = PR::ParticleReader::createReader(fileType, {INFILE2[n]}, myid, true); + if (fileType == "PSPhdf5") { + std::string path = CURDIR + INFILE2[0]; + psp2 = PR::ParticleReader::createReader(fileType, INFILE2, myid, true); + } else { + psp2 = PR::ParticleReader::createReader(fileType, {INFILE2[n]}, myid, true); + } final_time = psp2->CurrentTime(); psp2->SelectType(COMP); if (myid==0) { - std::cout << "File 2: " << INFILE2[n] << endl; + if (fileType == "PSPhdf5") + std::cout << "Path 2: " << path2 << endl; + else + std::cout << "File 2: " << INFILE2[n] << endl; std::cout << "Found dump at time: " << final_time << std::endl; std::cout << std::string(40, '-') << std::endl; } From d3f7e9f01fc068630b73de99c96b96eed8a733bf Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 2 Jun 2025 18:23:51 -0400 Subject: [PATCH 070/106] Option to add metadata to all HDF5 files --- src/OutHDF5.H | 5 +++-- src/OutHDF5.cc | 10 +++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index 32b602d83..c9702e0d6 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -16,7 +16,7 @@ @param checkpt sets checkpoint mode: write a real8 snapshot with a unique filename @param preserve prevents overwrite of snapshot files @param expconfig set to true stashes the YAML configuration file in the HDF5 file - + @param allmeta set to true writes all meta data to each HDF5 file */ class OutHDF5 : public Output { @@ -25,7 +25,7 @@ private: std::string filename; bool real4=true, real8=false, ids=true, gadget4=false, chkpt=false; - bool overwrite=false, expconfig=true, timer; + bool overwrite=false, expconfig=true, allmeta=false, timer; unsigned write_flags; int nbeg; void initialize(void); @@ -63,6 +63,7 @@ public: \param checkpt set to true for checkpoint mode \param preserve set to true prevents overwrite of snapshot files \param expconfig set to true stashes the YAML configuration file in the HDF5 file + \param allmeta set to true writes all meta data to each HDF5 file */ void Run(int nstep, int mstep, bool last); diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index a025139f8..c4a7ef292 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -35,7 +35,8 @@ OutHDF5::valid_keys = { "preserve", "H5compress", "H5chunk", - "expconfig" + "expconfig", + "allmeta" }; OutHDF5::OutHDF5(const YAML::Node& conf) : Output(conf) @@ -124,6 +125,9 @@ void OutHDF5::initialize() if (Output::conf["expconfig"]) expconfig = Output::conf["expconfig"].as(); + if (Output::conf["allmeta"]) + allmeta = Output::conf["allmeta"].as(); + // Default HDF5 compression is no compression. By default, // shuffle is on unless turned off manually. // @@ -298,7 +302,7 @@ void OutHDF5::RunGadget4(const std::string& path) // Only attach metadata to the first file // - if (myid==0) { + if (allmeta or myid==0) { int dp = 1; header.createAttribute("Flag_DoublePrecision", dp); @@ -535,7 +539,7 @@ void OutHDF5::RunPSP(const std::string& path) // Only attach metadata to first process // - if (myid==0) { + if (allmeta or myid==0) { int dp = real4 ? 0 : 1; writeScalar(header, "Flag_DoublePrecision", dp); From 38aa8d7177391869924e04235ec79834907de956 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 3 Jun 2025 14:50:38 -0400 Subject: [PATCH 071/106] Add directory organization of HDF5 snapshots (enabled by default) --- src/OutHDF5.H | 5 +- src/OutHDF5.cc | 156 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 143 insertions(+), 18 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index c9702e0d6..51e7399b3 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -17,6 +17,7 @@ @param preserve prevents overwrite of snapshot files @param expconfig set to true stashes the YAML configuration file in the HDF5 file @param allmeta set to true writes all meta data to each HDF5 file + @param directory set to true stores all HDF5 files in a named directory */ class OutHDF5 : public Output { @@ -25,12 +26,13 @@ private: std::string filename; bool real4=true, real8=false, ids=true, gadget4=false, chkpt=false; - bool overwrite=false, expconfig=true, allmeta=false, timer; + bool overwrite=false, expconfig=true, allmeta=false, directory=true, timer; unsigned write_flags; int nbeg; void initialize(void); std::vector masses; std::vector multim; + std::string hdf_dir, chkpt_dir; //! Valid keys for YAML configurations static const std::set valid_keys; @@ -64,6 +66,7 @@ public: \param preserve set to true prevents overwrite of snapshot files \param expconfig set to true stashes the YAML configuration file in the HDF5 file \param allmeta set to true writes all meta data to each HDF5 file + \param directory set to true stores all HDF5 files in a named directory */ void Run(int nstep, int mstep, bool last); diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index c4a7ef292..055b06ba7 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -128,6 +129,9 @@ void OutHDF5::initialize() if (Output::conf["allmeta"]) allmeta = Output::conf["allmeta"].as(); + if (Output::conf["directory"]) + directory = Output::conf["directory"].as(); + // Default HDF5 compression is no compression. By default, // shuffle is on unless turned off manually. // @@ -158,37 +162,69 @@ void OutHDF5::initialize() throw std::runtime_error("OutHDF5::initialize: error parsing YAML"); } + // Determine last file // if (restart && nbeg==0) { - // Only root node looks for files + // Only root node looks for paths // if (myid==0) { - for (nbeg=0; nbeg<100000; nbeg++) { - // Output name - // - ostringstream fname; + // First check for a directory + // + if (directory) { + std::ostringstream dname; + dname << outdir + << filename << "_" << setw(5) << setfill('0') << nbeg; + + std::filesystem::path dir_path = dname.str(); + + // Sanity check + if (not std::filesystem::is_directory(dir_path)) { + throw std::runtime_error("Component::initialize error: you specified directory organization of output but the directory " + dir_path.string() + " does not exist"); + } + } + else { + std::ostringstream fname; fname << outdir << filename << "_" << setw(5) << setfill('0') << nbeg << ".1"; - // See if we can open file - // - ifstream in(fname.str().c_str()); + std::filesystem::path file_path = fname.str(); - if (!in) { - cout << "OutHDF5: will begin with nbeg=" << nbeg << endl; - break; + if (not std::filesystem::is_regular_file(file_path)) { + throw std::runtime_error("Component::initialize error: you specified file organization of output but the file " + fname.str() + " does not exist"); } } + + // Find starting point + for (; nbeg<100000; nbeg++) { + // Path name + // + std::ostringstream fname; + fname << outdir + << filename << "_" << setw(5) << setfill('0') << nbeg; + + if (not directory) fname << ".1"; + + std::filesystem::path path = fname.str(); + + // See if we can open + // + if (directory) + if (not std::filesystem::is_directory(path)) break; + else + if (not std::filesystem::is_regular_file(path)) break; + } } // Communicate starting file to all nodes // MPI_Bcast(&nbeg, 1, MPI_INT, 0, MPI_COMM_WORLD); } + + return; } @@ -204,55 +240,141 @@ void OutHDF5::Run(int n, int mstep, bool last) if (timer) beg = std::chrono::high_resolution_clock::now(); std::ofstream out; + std::string h5_dir; std::ostringstream fname; // Checkpoint mode if (chkpt) { + + // On first call, create checkpoint directory + if (directory and chkpt_dir.size()==0) { + std::ostringstream dname; + dname << outdir << "checkpoint_" << runtag; + + std::filesystem::path dir_path = dname.str(); + + chkpt_dir = dir_path.string() + "/"; + + bool okay = true; + if (myid==0) { + if (not std::filesystem::is_directory(dir_path)) { + if (std::filesystem::create_directory(dir_path)) { + std::cout << "---- OutHDF5: checkpoint directory <" + << dir_path.string() << "> created successfully" + << std::endl; + okay = true; + } else { + std::cout << "---- OutHDF5: checkpoint directory <" + << dir_path.string() << "> creation failed" + << std::endl; + okay = false; + } + } + } + + MPI_Bcast(&okay, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + + if (not okay) + throw std::runtime_error + ("OutHDF5: Failed to create checkpoint directory " + + dir_path.string()); + + } + // Create checkpoint filename + // fname << "checkpoint_" << runtag; if (numprocs>1) fname << "." << myid+1; // Create backup filename - std::string currfile = outdir + fname.str(); + // + std::string currfile = chkpt_dir + fname.str(); std::string backfile = currfile + ".bak"; // Remove old backup file + // if (unlink(backfile.c_str())) { if (VERBOSE>5) perror("OutHDF5::Run()"); - std::cout << "OutHDF5::Run(): error unlinking old backup file <" + std::cout << "---- OutHDF5::Run(): error unlinking old backup file <" << backfile << ">, it may not exist" << std::endl; } else { if (VERBOSE>5) { - std::cout << "OutHDF5::Run(): successfully unlinked <" + std::cout << "---- OutHDF5::Run(): successfully unlinked <" << backfile << ">" << std::endl; } } // Rename current file to backup file + // if (rename(currfile.c_str(), backfile.c_str())) { if (VERBOSE>5) perror("OutHDF5::Run()"); - std::cout << "OutHDF5: renaming backup file <" + std::cout << "---- OutHDF5: renaming backup file <" << backfile << ">, it may not exist" << std::endl; } else { if (VERBOSE>5) { - std::cout << "OutHDF5::Run(): successfully renamed <" + std::cout << "---- OutHDF5::Run(): successfully renamed <" << currfile << "> to <" << backfile << ">" << std::endl; } } } // Standard snapshot mode else { + + // Create filename prefix + // fname << filename << "_" << setw(5) << setfill('0') << nbeg++; + + if (directory) { + bool okay = true; + + // The prefix becomes the directory name + // + std::ostringstream dname; + dname << outdir << fname.str(); + + // Root process creates the directory + // + if (myid==0) { + std::filesystem::path dir_path = dname.str(); + + if (not std::filesystem::is_directory(dir_path)) { + if (std::filesystem::create_directory(dir_path)) { + std::cout << "HDF5 directory <" << dir_path.string() + << "> created successfully" << std::endl; + okay = true; + } else { + std::cout << "HDF5 directory <" << dir_path.string() + << "> creation failed" << std::endl; + okay = false; + } + } + } + + MPI_Bcast(&okay, 1, MPI_C_BOOL, 0, MPI_COMM_WORLD); + + if (not okay) + throw std::runtime_error + ("OutHDF5: Failed to create HDF5 file directory " + dname.str()); + + h5_dir = fname.str() + "/"; + } if (numprocs>1) fname << "." << myid+1; } // Full path - std::string path = outdir + fname.str(); + std::string path; + + if (directory) { // Append directory mode path + if (chkpt) path = chkpt_dir; + else path = outdir + h5_dir; + } + path += fname.str(); // Append filename if (gadget4) RunGadget4(path); else RunPSP(path); chktimer.mark(); + if (VERBOSE>5) perror("OutHDF5::Run()"); dump_signal = 0; From 54547167e626816ec1f4c61a32e199c25f51dc22 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 3 Jun 2025 14:51:16 -0400 Subject: [PATCH 072/106] Enable PSPhdf5 input --- utils/PhaseSpace/diffpsp.cc | 112 ++++++++++++++++++++---------------- utils/PhaseSpace/pspmono.cc | 42 ++++++++++++-- 2 files changed, 98 insertions(+), 56 deletions(-) diff --git a/utils/PhaseSpace/diffpsp.cc b/utils/PhaseSpace/diffpsp.cc index 736e79783..7a52b03bd 100755 --- a/utils/PhaseSpace/diffpsp.cc +++ b/utils/PhaseSpace/diffpsp.cc @@ -155,6 +155,8 @@ main(int argc, char **argv) " OUTFILE.rp Pericentric radius [2d]\n" \ " OUTFILE.O1 Radial frequency [2d]\n" \ " OUTFILE.O2 Azimuthal frequency [2d]\n" \ + " OUTFILE.F1 Two dimensional DF info (1) [2d]\n" \ + " OUTFILE.F2 Two dimensional DF info (2) [2d]\n" \ " OUTFILE.DR Run mass, J, Delta J (R) [1d]\n" \ " OUTFILE.df1 One dimensional DF info (1) [1d]\n" \ " OUTFILE.df2 One dimensional DF info (2) [1d]\n" \ @@ -456,7 +458,7 @@ main(int argc, char **argv) // Open output file // - const int nfiles = 19; + const int nfiles = 21; const char *suffix[] = { ".DM", // Mass per bin (E, K) #0 ".DE", // Delta E(E, K) #1 @@ -469,14 +471,16 @@ main(int argc, char **argv) ".DF", // Delta DF (E, K) #8 ".ra", // Apocentric radius (E, K) #9 ".rp", // Pericentric radius (E, K) #10 - ".O1", // Apocentric radius (E, K) #11 - ".O2", // Pericentric radius (E, K) #12 - ".Df", // Delta DF/F (E, K) #13 - ".DN", // Counts per (E, K) bin #14 - ".DR", // Run mass, J, Delta J (R) #15 - ".chk", // Orbital element check #16 - ".df1", // One dimensional DF info #17 - ".df2" // One dimensional DF info #18 + ".O1", // Radial frequency (E, K) #11 + ".O2", // Azimuthal frequency (E, K) #12 + ".F1", // DF for PS 1 (E, K) #13 + ".F2", // DF for PS 2 (E, K) #14 + ".Df", // Delta DF/F (E, K) #15 + ".DN", // Counts per (E, K) bin #16 + ".DR", // Run mass, J, Delta J (R) #17 + ".chk", // Orbital element check #18 + ".df1", // One dimensional DF info #19 + ".df2" // One dimensional DF info #20 }; std::vector filename(nfiles); for (int i=0; i0.0) { - p_rec(out[8], I1, I2, histoF(i, j)*jfac/totMass); + p_rec(out[8 ], I1, I2, histoF(i, j)*jfac/totMass); + p_rec(out[13], I1, I2, histo1(i, j)*jfac/totMass); + p_rec(out[14], I1, I2, histo2(i, j)*jfac/totMass); } else { - p_rec(out[8], I1, I2, 0.0); + p_rec(out[8 ], I1, I2, 0.0); + p_rec(out[13], I1, I2, 0.0); + p_rec(out[14], I1, I2, 0.0); } - p_rec(out[13], I1, I2, histoDF(i, j)); - p_rec(out[14], I1, I2, histoC (i, j)); + p_rec(out[15], I1, I2, histoDF(i, j)); + p_rec(out[16], I1, I2, histoC (i, j)); } - if (not meshgrid) for (int k=0; k<15; k++) out[k] << endl; + if (not meshgrid) for (int k=0; k<17; k++) out[k] << endl; } if (CUMULATE) { @@ -1460,36 +1470,36 @@ main(int argc, char **argv) }; for (int j=0; j #include +#include +#include #include #include #include #include #include +#include + #include #include #include @@ -75,7 +78,7 @@ main(int argc, char **argv) options.add_options() ("h,help", "This help message") - ("F,filetype", "input file type (one of: PSPout, PSPspl, GadgetNative, GadgetHDF5)", + ("F,filetype", "input file type (one of: PSPout, PSPspl, GadgetNative, GadgetHDF5, PSPhdf5)", cxxopts::value(fileType)->default_value("PSPout")) ("RMIN", "Minimum model radius", cxxopts::value(RMIN)->default_value("0.0")) @@ -138,23 +141,52 @@ main(int argc, char **argv) // Per file weight // double weight = 1.0/INFILE.size(); + int nfiles = INFILE.size(); + + std::string path; + + if (fileType=="PSPhdf5") { + path = CURDIR + INFILE[0]; + std::filesystem::path dir_path = path; + if (std::filesystem::is_directory(dir_path)) { + INFILE.clear(); + try { + for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { + if (std::filesystem::is_regular_file(entry)) { + std::string file = entry.path().string(); + if (H5::H5File::isHdf5(file)) INFILE.push_back(file); + } + } + } catch (const std::filesystem::filesystem_error& e) { + std::cerr << "pspmono error: " << e.what() << std::endl; + } + } + + nfiles = 1; + } // Loop through files // - for (size_t n=0; nSelectType(COMP); time = psp->CurrentTime(); if (myid==0) { - std::cout << "File: " << INFILE[n] << std::endl; + if (fileType=="PSPhdf5") + std::cout << "Path: " << path << std::endl; + else + std::cout << "File: " << INFILE[n] << std::endl; std::cout << "Found dump at time: " << time << std::endl; } } From 2cca1434e42a122679c88ccede2cafdff49292c9 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 3 Jun 2025 16:38:56 -0400 Subject: [PATCH 073/106] Additional check on Orient restart --- src/Orient.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Orient.cc b/src/Orient.cc index 99597dc24..1d1dbdedf 100644 --- a/src/Orient.cc +++ b/src/Orient.cc @@ -177,8 +177,13 @@ Orient::Orient(int n, int nwant, int naccel, unsigned Oflg, unsigned Cflg, bool allRead = true; for (int i=0; i<3; i++) { - if (line.eof()) allRead = false; - for (int k; k<3; k++) line >> pseudo(k); + if (line.eof()) { + allRead = false; + break; + } + else { + for (int k; k<3; k++) line >> pseudo(k); + } } if (allRead) { if (accel) accel->add(time, pseudo, axis1); From 5190f5cf8503b88f31f5eba25302441c8b071827 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 3 Jun 2025 16:39:16 -0400 Subject: [PATCH 074/106] Ignore .bak files --- src/ComponentContainer.cc | 41 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/ComponentContainer.cc b/src/ComponentContainer.cc index 56dbcc060..5bfe91778 100644 --- a/src/ComponentContainer.cc +++ b/src/ComponentContainer.cc @@ -110,17 +110,55 @@ void ComponentContainer::initialize(void) if (ih<1) throw std::runtime_error("ComponentContainer::initialize HDF5 restart directory found but no HDF5 files found"); + auto hasEnding = [](const std::string& fullStr, + const std::string& ending) -> bool + { + if (fullStr.length() >= ending.length()) { + return fullStr.compare(fullStr.length() - ending.length(), + ending.length(), ending) == 0; + } else { + return false; + } + }; + std::filesystem::path dir_path = outdir + infile; std::vector files; try { for (const auto& entry : std::filesystem::directory_iterator(dir_path)) { auto file = entry.path().string(); - if (H5::H5File::isHdf5(file)) files.push_back(file); + if (H5::H5File::isHdf5(file)) { + // Ignore checkpoint backup files + if (not hasEnding(file, ".bak")) files.push_back(file); + } } } catch (const std::filesystem::filesystem_error& e) { std::cerr << "ComponentContainer::initialize HDF5 file listing error: " << e.what() << std::endl; } + // Need to sort these files so that ".1" is first in the list so + // we can elide the metadata from the other HDF5 files + if (files.size() > 1) { + + // Function to extract the numerical substring and convert to + // an integer + auto getIndex = [](const std::string& str) -> int + { + auto pos = str.find_last_of("."); // File ends in .int + if (pos == std::string::npos) return 0; // No index + else return std::stoi(str.substr(pos+1)); // Extract index + }; + + // Custom comparison function + auto compareStringsByIndex = + [&getIndex](const std::string& a, const std::string& b) -> bool + { + return getIndex(a) < getIndex(b); + }; + + // Sort the files by index + std::sort(files.begin(), files.end(), compareStringsByIndex); + } + PR::PSPhdf5 reader(files, true); auto types = reader.GetTypes(); @@ -128,7 +166,6 @@ void ComponentContainer::initialize(void) if (not ignore_info) tnow = reader.CurrentTime(); - if (myid==0) { if (ignore_info) { cout << "---- ComponentContainer found: " From 89e7407cb07ec1e92e8c6c9a3278dd00235b7b13 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 3 Jun 2025 16:39:58 -0400 Subject: [PATCH 075/106] Comments and output changes only --- src/OutHDF5.cc | 55 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 055b06ba7..e1e9096ae 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -166,7 +166,7 @@ void OutHDF5::initialize() // Determine last file // if (restart && nbeg==0) { - + // Only root node looks for paths // if (myid==0) { @@ -198,7 +198,8 @@ void OutHDF5::initialize() } } - // Find starting point + // Find starting index + // for (; nbeg<100000; nbeg++) { // Path name // @@ -210,16 +211,18 @@ void OutHDF5::initialize() std::filesystem::path path = fname.str(); - // See if we can open + // See if we can open the directory or file // - if (directory) + if (directory) { if (not std::filesystem::is_directory(path)) break; - else + } else if (not std::filesystem::is_regular_file(path)) break; } + + std::cout << "---- OutHDF5: found last file <" << nbeg << ">" << std::endl; } - // Communicate starting file to all nodes + // Communicate starting file index to all nodes // MPI_Bcast(&nbeg, 1, MPI_INT, 0, MPI_COMM_WORLD); } @@ -259,14 +262,16 @@ void OutHDF5::Run(int n, int mstep, bool last) if (myid==0) { if (not std::filesystem::is_directory(dir_path)) { if (std::filesystem::create_directory(dir_path)) { - std::cout << "---- OutHDF5: checkpoint directory <" - << dir_path.string() << "> created successfully" - << std::endl; + if (VERBOSE>5) + std::cout << "---- OutHDF5: checkpoint directory <" + << dir_path.string() << "> created successfully" + << std::endl; okay = true; } else { - std::cout << "---- OutHDF5: checkpoint directory <" - << dir_path.string() << "> creation failed" - << std::endl; + if (VERBOSE>5) + std::cout << "---- OutHDF5: checkpoint directory <" + << dir_path.string() << "> creation failed" + << std::endl; okay = false; } } @@ -294,9 +299,11 @@ void OutHDF5::Run(int n, int mstep, bool last) // Remove old backup file // if (unlink(backfile.c_str())) { - if (VERBOSE>5) perror("OutHDF5::Run()"); - std::cout << "---- OutHDF5::Run(): error unlinking old backup file <" - << backfile << ">, it may not exist" << std::endl; + if (VERBOSE>5) { + perror("OutHDF5::Run()"); + std::cout << "---- OutHDF5::Run(): error unlinking old backup file <" + << backfile << ">, it may not exist" << std::endl; + } } else { if (VERBOSE>5) { std::cout << "---- OutHDF5::Run(): successfully unlinked <" @@ -307,9 +314,11 @@ void OutHDF5::Run(int n, int mstep, bool last) // Rename current file to backup file // if (rename(currfile.c_str(), backfile.c_str())) { - if (VERBOSE>5) perror("OutHDF5::Run()"); - std::cout << "---- OutHDF5: renaming backup file <" - << backfile << ">, it may not exist" << std::endl; + if (VERBOSE>5) { + perror("OutHDF5::Run()"); + std::cout << "---- OutHDF5: renaming backup file <" + << backfile << ">, it may not exist" << std::endl; + } } else { if (VERBOSE>5) { std::cout << "---- OutHDF5::Run(): successfully renamed <" @@ -339,12 +348,14 @@ void OutHDF5::Run(int n, int mstep, bool last) if (not std::filesystem::is_directory(dir_path)) { if (std::filesystem::create_directory(dir_path)) { - std::cout << "HDF5 directory <" << dir_path.string() - << "> created successfully" << std::endl; + if (VERBOSE>5) + std::cout << "HDF5 directory <" << dir_path.string() + << "> created successfully" << std::endl; okay = true; } else { - std::cout << "HDF5 directory <" << dir_path.string() - << "> creation failed" << std::endl; + if (VERBOSE>5) + std::cout << "HDF5 directory <" << dir_path.string() + << "> creation failed" << std::endl; okay = false; } } From aae2f6224baef31fb1d5467d6904901568a1e913 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 3 Jun 2025 18:47:34 -0400 Subject: [PATCH 076/106] Add missing PSP pot and potext fields --- exputil/ParticleReader.cc | 12 +++++++++--- src/Component.cc | 10 +++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 44efe8e4f..226485d2f 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -789,7 +789,7 @@ namespace PR { Eigen::Matrix pos, vel; std::vector idx; - std::vector mas; + std::vector mas, pot, potext; Eigen::Matrix iattrib; Eigen::Matrix rattrib; @@ -799,8 +799,11 @@ namespace PR { if (mass[curindx] == 0) part.getDataSet("Masses").read(mas); - part.getDataSet("Coordinates").read(pos); - part.getDataSet("Velocities" ).read(vel); + part.getDataSet("Coordinates" ).read(pos); + part.getDataSet("Velocities" ).read(vel); + + part.getDataSet("Potential" ).read(pot); + part.getDataSet("PotentialExt").read(potext); if (Niattrib[curindx]>0) part.getDataSet("IntAttributes" ).read(iattrib); @@ -831,6 +834,9 @@ namespace PR { P.vel[k] = vel(n, k); } + P.pot = pot[n]; + P.potext = potext[n]; + if (Niattrib[curindx]>0) { P.iattrib.resize(Niattrib[curindx]); for (int j=0; j pos(nbodies, 3); Eigen::Matrix vel(nbodies, 3); - std::vector mas; + std::vector mas, pot, potext; std::vector ids; Eigen::Matrix iattrib; @@ -2472,6 +2472,9 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) if (niattrib) iattrib.resize(nbodies, niattrib); if (ndattrib) dattrib.resize(nbodies, niattrib); + pot. resize(nbodies); + potext.resize(nbodies); + unsigned int ctr = 0; for (auto it=particles.begin(); it!=particles.end(); it++) { @@ -2483,6 +2486,8 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) if (IDs) ids[ctr] = P->indx; for (int j=0; j<3; j++) pos(ctr, j) = P->pos[j]; for (int j=0; j<3; j++) vel(ctr, j) = P->vel[j]; + pot [ctr] = P->pot; + potext[ctr] = P->potext; for (int j=0; jiattrib[j]; for (int j=0; jdattrib[j]; @@ -2536,6 +2541,9 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) HighFive::DataSet dsPos = group.createDataSet("Coordinates", pos, dcpl3); HighFive::DataSet dsVel = group.createDataSet("Velocities", vel, dcpl3); + HighFive::DataSet dsPot = group.createDataSet("Potential", pot, dcpl1); + HighFive::DataSet dsExt = group.createDataSet("PotentialExt", potext, dcpl1); + if (niattrib>0) HighFive::DataSet dsInt = group.createDataSet("IntAttributes", iattrib, dcplI); From 55232f4cbaba1242b9e4caf76834fd096bd68713 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 5 Jun 2025 11:14:14 -0400 Subject: [PATCH 077/106] Pretty-up some log output only --- src/OutHDF5.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index e1e9096ae..3093a5c77 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -300,7 +300,7 @@ void OutHDF5::Run(int n, int mstep, bool last) // if (unlink(backfile.c_str())) { if (VERBOSE>5) { - perror("OutHDF5::Run()"); + perror("---- OutHDF5::Run() perror"); std::cout << "---- OutHDF5::Run(): error unlinking old backup file <" << backfile << ">, it may not exist" << std::endl; } @@ -315,7 +315,7 @@ void OutHDF5::Run(int n, int mstep, bool last) // if (rename(currfile.c_str(), backfile.c_str())) { if (VERBOSE>5) { - perror("OutHDF5::Run()"); + perror("---- OutHDF5::Run() perror"); std::cout << "---- OutHDF5: renaming backup file <" << backfile << ">, it may not exist" << std::endl; } @@ -385,7 +385,7 @@ void OutHDF5::Run(int n, int mstep, bool last) else RunPSP(path); chktimer.mark(); - if (VERBOSE>5) perror("OutHDF5::Run()"); + if (VERBOSE>5) perror("---- OutHDF5::Run() perror"); dump_signal = 0; From 529003c67b8f9eeaad3959d4f27e63b8b568aefa Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 11 Jun 2025 17:25:26 -0400 Subject: [PATCH 078/106] Fix coefficient storage allocation error --- expui/BiorthBasis.cc | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/expui/BiorthBasis.cc b/expui/BiorthBasis.cc index 93f73a336..7b0cb6879 100644 --- a/expui/BiorthBasis.cc +++ b/expui/BiorthBasis.cc @@ -294,7 +294,7 @@ namespace BasisClasses 0, 1, cachename); // Test basis for consistency - orthoTest(200); + if (myid==0) orthoTest(200); } void Bessel::initialize() @@ -320,7 +320,7 @@ namespace BasisClasses bess = std::make_shared(rmax, lmax, nmax, rnum); // Test basis for consistency - orthoTest(200); + if (myid==0) orthoTest(200); } void Spherical::reset_coefs(void) @@ -1362,7 +1362,7 @@ namespace BasisClasses // Orthogonality sanity check // - orthoTest(); + if (myid==0) orthoTest(); // Set cylindrical coordindates // @@ -1549,7 +1549,6 @@ namespace BasisClasses "nmaxfid", "rcylmin", "rcylmax", - "acyltbl", "numx", "numy", "numr", @@ -1667,10 +1666,8 @@ namespace BasisClasses // Set characteristic radius defaults // - if (not conf["acyltbl"]) conf["acyltbl"] = 0.6; if (not conf["scale"]) conf["scale"] = 1.0; - // Check for non-null cache file name. This must be specified // to prevent recomputation and unexpected behavior. // @@ -1689,7 +1686,7 @@ namespace BasisClasses // Orthogonality sanity check // - orthoTest(); + if (myid==0) orthoTest(); // Get max threads // @@ -1735,9 +1732,12 @@ namespace BasisClasses cf->nmax = nmax; cf->time = time; - cf->store((2*mmax+1)*nmax); + // Allocate the coefficient storage + cf->store.resize((mmax+1)*nmax); + + // Make the coefficient map cf->coefs = std::make_shared - (cf->store.data(), 2*mmax+1, nmax); + (cf->store.data(), mmax+1, nmax); for (int m=0, m0=0; m<=mmax; m++) { for (int n=0; nnmax = nmax; cf->time = time; - cf->store((2*mmax+1)*nmax); + // Allocate the coefficient storage + cf->store.resize((mmax+1)*nmax); + + // Make the coefficient map cf->coefs = std::make_shared - (cf->store.data(), 2*mmax+1, nmax); + (cf->store.data(), mmax+1, nmax); for (int m=0, m0=0; m<=mmax; m++) { for (int n=0; n Date: Mon, 16 Jun 2025 12:11:39 -0400 Subject: [PATCH 079/106] Throw runtime exception if the user tries to write an HDF5 coefficient file from an empty Coefs instance --- expui/Coefficients.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/expui/Coefficients.cc b/expui/Coefficients.cc index 52d8f69e7..b01cb0c04 100644 --- a/expui/Coefficients.cc +++ b/expui/Coefficients.cc @@ -2573,6 +2573,18 @@ namespace CoefClasses void Coefs::WriteH5Coefs(const std::string& prefix) { + // Sanity check: throw runtime error if there are no coefficient + // sets + // + if (Times().size() == 0) { + throw std::runtime_error + ("Coefs::WriteH5Coefs: " + "we have NO coefficient sets...continuing without writing" + ); + } + + // Write coefficients + // try { // Create a new hdf5 file // From 29ade3225b8506b461a7e4ce348775b70e880551 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 18 Jun 2025 10:45:59 -0400 Subject: [PATCH 080/106] Generalize histo1dlog to return velocity dispersion --- expui/FieldGenerator.H | 2 +- expui/FieldGenerator.cc | 38 +++++++++++++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/expui/FieldGenerator.H b/expui/FieldGenerator.H index f342d5dc5..65b2624ae 100644 --- a/expui/FieldGenerator.H +++ b/expui/FieldGenerator.H @@ -119,7 +119,7 @@ namespace Field std::vector center={0.0, 0.0, 0.0}); //! Compute spherical log histogram from particles - std::tuple + std::tuple histo1dlog(PR::PRptr reader, double rmin, double rmax, int nbins, std::vector center={0.0, 0.0, 0.0}); diff --git a/expui/FieldGenerator.cc b/expui/FieldGenerator.cc index c79c180b9..a71470dda 100644 --- a/expui/FieldGenerator.cc +++ b/expui/FieldGenerator.cc @@ -919,7 +919,7 @@ namespace Field } // END histogram1d - std::tuple + std::tuple FieldGenerator::histo1dlog(PR::PRptr reader, double rmin, double rmax, int nbins, std::vector ctr) { @@ -930,6 +930,9 @@ namespace Field Eigen::VectorXf rad = Eigen::VectorXf::Zero(nbins); Eigen::VectorXf ret = Eigen::VectorXf::Zero(nbins); + Eigen::VectorXf vel = Eigen::VectorXf::Zero(nbins); + Eigen::MatrixXf vc2 = Eigen::MatrixXf::Zero(nbins, 3); + Eigen::MatrixXf vc1 = Eigen::MatrixXf::Zero(nbins, 3); double lrmin = log(rmin), lrmax = log(rmax); double del = (lrmax - lrmin)/nbins; @@ -944,18 +947,35 @@ namespace Field } int indx = floor((log(sqrt(rad)) - lrmin)/del); - if (indx>=0 and indxmass; + if (indx>=0 and indxmass; + for (int k=0; k<3; k++) { + double v = p->vel[k]; + vc1(indx, k) += p->mass * v; + vc2(indx, k) += p->mass * v*v; + } + } } // Accumulate between MPI nodes; return value to root node // if (use_mpi) { - if (myid==0) + if (myid==0) { MPI_Reduce(MPI_IN_PLACE, ret.data(), ret.size(), MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); - else + MPI_Reduce(MPI_IN_PLACE, vc1.data(), vc1.size(), + MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(MPI_IN_PLACE, vc2.data(), vc2.size(), + MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); + } + else { MPI_Reduce(ret.data(), NULL, ret.size(), MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(vc1.data(), NULL, vc1.size(), + MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); + MPI_Reduce(vc2.data(), NULL, vc2.size(), + MPI_FLOAT, MPI_SUM, 0, MPI_COMM_WORLD); + } } // Compute density @@ -963,11 +983,19 @@ namespace Field double d3 = exp(3.0*del); double rf = 4.0*pi/3.0*(d3 - 1.0); for (int i=0; i Date: Wed, 18 Jun 2025 10:46:46 -0400 Subject: [PATCH 081/106] Add static functions to read and parse HDF5 directories to save the user the trouble of having to pass full file lists (which will still work) --- exputil/ParticleReader.cc | 85 ++++++++++++++++++++++++++++++++++++++- include/ParticleReader.H | 3 ++ 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 226485d2f..c06e8da92 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,6 +8,7 @@ #include #include #include +#include // HighFive API #include #include @@ -638,17 +640,66 @@ namespace PR { else return 0; } } + + std::vector + ParticleReader::scanDirectory(const std::string& dir) + { + // Will contain the filenames from the directory scan + std::vector ret; + + // Check if the directory exists + std::filesystem::path p(dir); + + // If the directory does not exist, ret will be returned with zero + // length + if (std::filesystem::is_directory(p)) { + + // Iterate over the directory entries + for (const auto& entry : std::filesystem::directory_iterator(p)) { + + // Check if the entry is a regular file + if (std::filesystem::is_regular_file(entry.status())) { + + // Get the filename + std::string filename = entry.path().string(); + + // Check if the last character is a digit + if (std::isdigit(filename.back())) { + + // Assume that this is a snapshot file. We could check if + // this is HDF5 here but I'd rather that this procedure be + // file-type agnostic. + ret.push_back(filename); + } + } + } + } + + return ret; + } - + PSPhdf5::PSPhdf5(const std::vector& files, bool verbose) { _files = files; _verbose = verbose; + // Do we have a directory of snapshot files? + // + if (_files.size()==1) { + std::vector fscan = scanDirectory(_files[0]); + if (fscan.size() != 0) _files = fscan; + // Put the first file at the top of the list + std::partial_sort(_files.begin(), _files.begin()+1, _files.end()); + } + + // Read metadata from the first file + // getInfo(); // Sanity check - if (nfiles != files.size()) + // + if (nfiles != _files.size()) throw GenericError("PSPhdf5: number of files does not match number expected for this snapshot", __FILE__, __LINE__, 1042, true); curfile = _files.begin(); @@ -1832,6 +1883,28 @@ namespace PR { return parseStringList(files, delimit); } + // Are all the files in the list directories? + bool + ParticleReader::parseDirectoryList(const std::vector& files) + { + int dcount = 0; // The directory count + int fcount = 0; // The file count + + for (auto f : files) { + if (std::filesystem::is_directory(f)) + dcount++; + else + fcount++; + } + + if (dcount>0 and fcount>0) + throw std::runtime_error("ParticleReader::parseDirectoryList: " + "cannot mix directories and files"); + + return dcount>0; + } + + std::vector> ParticleReader::parseStringList (const std::vector& infiles, const std::string& delimit) @@ -1842,6 +1915,14 @@ namespace PR { auto files = infiles; std::sort(files.begin(), files.end()); + // Check to see if these are all directories + if (parseDirectoryList(files)) { + for (auto d : files) { + batches.push_back(std::vector{d}); + } + return batches; + } + std::vector batch; std::string templ; diff --git a/include/ParticleReader.H b/include/ParticleReader.H index 3c835dbf1..0f60923ed 100644 --- a/include/ParticleReader.H +++ b/include/ParticleReader.H @@ -36,6 +36,9 @@ namespace PR int numprocs, myid; bool use_mpi; + static bool parseDirectoryList(const std::vector& files); + static std::vector scanDirectory(const std::string& dir); + public: //! Constructor: check for and set up MPI From d2492bea72e2b3e9f2541481bcb8618dfbd836f9 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 18 Jun 2025 13:01:20 -0400 Subject: [PATCH 082/106] Additional commenting only --- exputil/ParticleReader.cc | 27 ++++++++++++++++++--------- include/ParticleReader.H | 4 ++++ 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index c06e8da92..63722be38 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -644,14 +644,14 @@ namespace PR { std::vector ParticleReader::scanDirectory(const std::string& dir) { - // Will contain the filenames from the directory scan + // 'ret' will contain the filenames from the directory scan std::vector ret; - // Check if the directory exists + // The directory path std::filesystem::path p(dir); - // If the directory does not exist, ret will be returned with zero - // length + // Check if the directory exists. If the directory does not + // exist, then 'ret' will be returned with zero length if (std::filesystem::is_directory(p)) { // Iterate over the directory entries @@ -660,10 +660,12 @@ namespace PR { // Check if the entry is a regular file if (std::filesystem::is_regular_file(entry.status())) { - // Get the filename + // Get the full path name std::string filename = entry.path().string(); - // Check if the last character is a digit + // Check if the last character is a digit, consistent with a + // partial phase-space write and not a backup or metadata + // file if (std::isdigit(filename.back())) { // Assume that this is a snapshot file. We could check if @@ -688,9 +690,16 @@ namespace PR { // if (_files.size()==1) { std::vector fscan = scanDirectory(_files[0]); - if (fscan.size() != 0) _files = fscan; - // Put the first file at the top of the list - std::partial_sort(_files.begin(), _files.begin()+1, _files.end()); + + // Did we find files? + // + if (fscan.size() != 0) { + _files = fscan; + + // Put the first file at the top of the list for metadata + // reading + std::partial_sort(_files.begin(), _files.begin()+1, _files.end()); + } } // Read metadata from the first file diff --git a/include/ParticleReader.H b/include/ParticleReader.H index 0f60923ed..0d150dc45 100644 --- a/include/ParticleReader.H +++ b/include/ParticleReader.H @@ -36,7 +36,11 @@ namespace PR int numprocs, myid; bool use_mpi; + //! Check for a list of directories or files and not both static bool parseDirectoryList(const std::vector& files); + + //! Scan a directory for partial phase-space writes ending in a + //! integer index static std::vector scanDirectory(const std::string& dir); public: From 5d5874ece698230ac2469936e2b033fe9bb94bd4 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 18 Jun 2025 19:22:50 -0400 Subject: [PATCH 083/106] Added directory file listing to Gadget and Tipsy readers --- exputil/ParticleReader.cc | 41 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 63722be38..ba998aa9d 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -40,6 +40,18 @@ namespace PR { _files = files; // Copy file list (bunch) _verbose = verbose; + // Do we have a directory of snapshot files? + // + if (_files.size()==1) { + std::vector fscan = scanDirectory(_files[0]); + + // Did we find files? + // + if (fscan.size() != 0) { + _files = fscan; + } + } + ptype = 1; // Default is halo particles getNumbers(); // Get the number of particles in all @@ -310,6 +322,18 @@ namespace PR { _files = files; _verbose = verbose; + // Do we have a directory of snapshot files? + // + if (_files.size()==1) { + std::vector fscan = scanDirectory(_files[0]); + + // Did we find files? + // + if (fscan.size() != 0) { + _files = fscan; + } + } + ptype = 1; // Default is halo particles totalCount = 0; // Initialization of particles read @@ -318,7 +342,7 @@ namespace PR { curfile = _files.begin(); if (not nextFile()) { - std::cerr << "GadgetNative: no files found" << std::endl; + std::cerr << "GadgetHDF5: no files found" << std::endl; } } @@ -2115,8 +2139,21 @@ namespace PR { Tipsy::Tipsy(const std::vector& filelist, TipsyType Type, bool verbose) { - ttype = Type; files = filelist; + + // Do we have a directory of snapshot files? + // + if (files.size()==1) { + std::vector fscan = scanDirectory(files[0]); + + // Did we find files? + // + if (fscan.size() != 0) { + files = fscan; + } + } + + ttype = Type; getNumbers(); curfile = files.begin(); if (not nextFile()) { From eb2714ccb250cde62328ecf75e7187616e899340 Mon Sep 17 00:00:00 2001 From: mdw Date: Fri, 20 Jun 2025 09:51:36 -0400 Subject: [PATCH 084/106] Use correct type in MPI_Bcast; it's nuts that there isn't a proper C++ binding with autodeduction --- src/ComponentContainer.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ComponentContainer.cc b/src/ComponentContainer.cc index 5bfe91778..3f50d6c0a 100644 --- a/src/ComponentContainer.cc +++ b/src/ComponentContainer.cc @@ -93,10 +93,10 @@ void ComponentContainer::initialize(void) } } - MPI_Bcast(&ir, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); - MPI_Bcast(&is, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); - MPI_Bcast(&ih, 1, MPI_UNSIGNED_CHAR, 0, MPI_COMM_WORLD); - MPI_Bcast(&HDF5, 1, MPI_CXX_BOOL, 0, MPI_COMM_WORLD); + MPI_Bcast(&ir, 1, MPI_UNSIGNED_SHORT, 0, MPI_COMM_WORLD); + MPI_Bcast(&is, 1, MPI_UNSIGNED_SHORT, 0, MPI_COMM_WORLD); + MPI_Bcast(&ih, 1, MPI_UNSIGNED_SHORT, 0, MPI_COMM_WORLD); + MPI_Bcast(&HDF5, 1, MPI_CXX_BOOL, 0, MPI_COMM_WORLD); restart = ir ? true : false; SPL = is ? true : false; From 33ba3781c097800e9bafdd1ebd087688c01a07f6 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Fri, 20 Jun 2025 10:02:52 -0400 Subject: [PATCH 085/106] Additional comments describing restart control only [No CI] --- src/ComponentContainer.cc | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/ComponentContainer.cc b/src/ComponentContainer.cc index 3f50d6c0a..8f95a49d3 100644 --- a/src/ComponentContainer.cc +++ b/src/ComponentContainer.cc @@ -51,10 +51,12 @@ void ComponentContainer::initialize(void) bool SPL = false; // Indicates whether file has the SPL prefix bool HDF5 = false; // Indicates whether file is an HDF5 file - unsigned short ir = 0; - unsigned short is = 0; - unsigned short ih = 0; - // Look for a restart file + unsigned short ir = 0; // Number of restart files + unsigned short is = 0; // Number of SPL files + unsigned short ih = 0; // Number of HDF5 files + + // Look for a restart file + // if (myid==0) { std::string resfile = outdir + infile; @@ -93,14 +95,21 @@ void ComponentContainer::initialize(void) } } + // Share file counts and HDF5 detection + // MPI_Bcast(&ir, 1, MPI_UNSIGNED_SHORT, 0, MPI_COMM_WORLD); MPI_Bcast(&is, 1, MPI_UNSIGNED_SHORT, 0, MPI_COMM_WORLD); MPI_Bcast(&ih, 1, MPI_UNSIGNED_SHORT, 0, MPI_COMM_WORLD); MPI_Bcast(&HDF5, 1, MPI_CXX_BOOL, 0, MPI_COMM_WORLD); + // Set restart flags. 'restart' is an EXP global. 'SPL' and 'HDF5' + // are local to this member function. + // restart = ir ? true : false; SPL = is ? true : false; + // Begin phase space recovery + // if (restart) { if (HDF5) { From 21d28fec2f2388dd53092844d1366a5b5c683348 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 1 Jul 2025 12:12:12 -0400 Subject: [PATCH 086/106] Output log changes for ortho test results only --- expui/BiorthBasis.H | 7 +++++-- exputil/orthoTest.cc | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/expui/BiorthBasis.H b/expui/BiorthBasis.H index 045e209e7..59e8bb26a 100644 --- a/expui/BiorthBasis.H +++ b/expui/BiorthBasis.H @@ -841,8 +841,11 @@ namespace BasisClasses bool orthoTest() { auto [ret, worst, lworst] = orthoCompute(sl->orthoCheck()); - // For the CTest log - std::cout << "---- Cylindrical::orthoTest: worst=" << worst << std::endl; + if (myid==0) { + // For the CTest log + std::cout << "---- Cylindrical::orthoTest: worst=" << worst + << std::endl; + } return ret; } }; diff --git a/exputil/orthoTest.cc b/exputil/orthoTest.cc index 99fcb4f2e..00a16738d 100644 --- a/exputil/orthoTest.cc +++ b/exputil/orthoTest.cc @@ -81,6 +81,7 @@ void orthoTest(const std::vector& tests, } else { // Success message if (myid==0) - std::cout << classname + ": biorthogonal check passed" << std::endl; + std::cout << "---- " << classname + ": biorthogonal check passed" + << std::endl; } } From 059b231bead7e9edd63d554336de8b48502e7685 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 1 Jul 2025 13:22:45 -0400 Subject: [PATCH 087/106] Another minor output log change for ortho test results only --- exputil/EmpCylSL.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/exputil/EmpCylSL.cc b/exputil/EmpCylSL.cc index 1e631c7a9..9024c0e47 100644 --- a/exputil/EmpCylSL.cc +++ b/exputil/EmpCylSL.cc @@ -399,7 +399,7 @@ void EmpCylSL::reset(int numr, int lmax, int mmax, int nord, ortho = std::make_shared(make_sl(), LMAX, NMAX, NUMR, RMIN, RMAX*0.99, false, 1, 1.0); - orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL[SLGridSph]", "l"); + orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL [SLGridSph]", "l"); // Resize (should not be necessary) but just in case some future // feature changes mulitstep on the fly @@ -878,7 +878,7 @@ int EmpCylSL::read_eof_file(const string& eof_file) ortho = std::make_shared(make_sl(), LMAX, NMAX, NUMR, RMIN, RMAX*0.99, false, 1, 1.0); - orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL[SLGridSph]", "l"); + orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL [SLGridSph]", "l"); setup_eof(); setup_accumulation(); @@ -1450,7 +1450,7 @@ void EmpCylSL::compute_eof_grid(int request_id, int m) ortho = std::make_shared(make_sl(), LMAX, NMAX, NUMR, RMIN, RMAX*0.99, false, 1, 1.0); - orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL[SLGridSph]", "l"); + orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL [SLGridSph]", "l"); } @@ -1633,7 +1633,7 @@ void EmpCylSL::compute_even_odd(int request_id, int m) LMAX, NMAX, NUMR, RMIN, RMAX*0.99, false, 1, 1.0); - orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL[SLGridSph]", "l"); + orthoTest(ortho->orthoCheck(std::max(NMAX*50, 200)), "EmpCylSL [SLGridSph]", "l"); } double dens, potl, potr, pott; From 43331c5ad8a588198deab325b136b84d07562498 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Thu, 10 Jul 2025 16:38:42 -0400 Subject: [PATCH 088/106] Add Output configuration parameters to 'current_keys' list in base class constructor to enable valid parameter checking --- src/Output.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Output.cc b/src/Output.cc index b48744ded..10c4a84e9 100644 --- a/src/Output.cc +++ b/src/Output.cc @@ -4,6 +4,13 @@ Output::Output(const YAML::Node& CONF) : conf(CONF) { - nint = 50; // Default interval + // Default interval + nint = 50; nintsub = std::numeric_limits::max(); + + // Add keys + for (YAML::const_iterator it=conf.begin(); it!=conf.end(); ++it) { + current_keys.insert(it->first.as()); + } + } From 7f05e2cdfb292f7aa38c10bb15e33cb9123578b6 Mon Sep 17 00:00:00 2001 From: Michael Petersen Date: Tue, 22 Jul 2025 12:58:33 +0100 Subject: [PATCH 089/106] Update exp.cfg to match 7.9.0 --- doc/exp.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/exp.cfg b/doc/exp.cfg index eb9d53c5c..8cb388127 100644 --- a/doc/exp.cfg +++ b/doc/exp.cfg @@ -48,7 +48,7 @@ PROJECT_NAME = EXP # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 7.8.1 +PROJECT_NUMBER = 7.9.0 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From c8791e763f99309f4322adc9d7803d3414bcbe27 Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Tue, 22 Jul 2025 09:00:21 -0400 Subject: [PATCH 090/106] Update src/Orient.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Orient.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Orient.cc b/src/Orient.cc index 1d1dbdedf..afbfb2ffa 100644 --- a/src/Orient.cc +++ b/src/Orient.cc @@ -182,7 +182,7 @@ Orient::Orient(int n, int nwant, int naccel, unsigned Oflg, unsigned Cflg, break; } else { - for (int k; k<3; k++) line >> pseudo(k); + for (int k=0; k<3; k++) line >> pseudo(k); } } if (allRead) { From 50bb248bb575aab6b1d29af8e1094816d9e0f7de Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Tue, 22 Jul 2025 09:00:33 -0400 Subject: [PATCH 091/106] Update src/OutHDF5.H Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/OutHDF5.H | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/OutHDF5.H b/src/OutHDF5.H index 51e7399b3..c48f1337d 100644 --- a/src/OutHDF5.H +++ b/src/OutHDF5.H @@ -1,8 +1,6 @@ #ifndef _OutHDF5_H #define _OutHDF5_H -#include - /** Write phase-space dumps at regular intervals from each node in component pieces. This writer is indended to be Gadget-4 HDF5 compliant. From e4edbe6669d2a836ba4d6570ff5b50b4cc06619d Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Tue, 22 Jul 2025 09:01:01 -0400 Subject: [PATCH 092/106] Update src/OutHDF5.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/OutHDF5.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 3093a5c77..48df0956d 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -719,7 +719,7 @@ void OutHDF5::RunPSP(const std::string& path) std::string gcommit(GIT_COMMIT), gbranch(GIT_BRANCH), gdate(COMPILE_TIME); writeScalar(params, "Git_commit", gcommit); writeScalar(params, "Git_branch", gbranch); - writeScalar(params, "Compile_data", gdate ); + writeScalar(params, "Compile_date", gdate ); std::vector names, forces, configs; for (auto c : comp->components) { From 728caab2bf7e5847968213d81c4740c239751589 Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Tue, 22 Jul 2025 09:02:04 -0400 Subject: [PATCH 093/106] Update exputil/ParticleReader.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- exputil/ParticleReader.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index ba998aa9d..22bb18819 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -927,7 +927,7 @@ namespace PR { } if (Ndattrib[curindx]>0) { - P.iattrib.resize(Ndattrib[curindx]); + P.dattrib.resize(Ndattrib[curindx]); for (int j=0; j Date: Tue, 22 Jul 2025 09:02:17 -0400 Subject: [PATCH 094/106] Update exputil/ParticleReader.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- exputil/ParticleReader.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exputil/ParticleReader.cc b/exputil/ParticleReader.cc index 22bb18819..ea63c0400 100644 --- a/exputil/ParticleReader.cc +++ b/exputil/ParticleReader.cc @@ -1182,7 +1182,7 @@ namespace PR { } if (Ndattrib[curindx]>0) { - P.iattrib.resize(Ndattrib[curindx]); + P.dattrib.resize(Ndattrib[curindx]); // Access data by pointer Scalar* ptr = (Scalar*)h5part[n].dattrib.p; for (int j=0; j Date: Tue, 22 Jul 2025 10:14:44 -0400 Subject: [PATCH 095/106] Do an all-process reduction rather than look for masses on a sngle processes --- src/OutHDF5.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 48df0956d..2f55672c5 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -794,8 +794,8 @@ void OutHDF5::checkParticleMasses() p = c->get_particles(&number); } - MPI_Bcast(&minMass, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); - MPI_Bcast(&maxMass, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &minMass, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &maxMass, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); if ( (maxMass - minMass)/maxMass < 1.0e-12) { masses.push_back(maxMass); From c5018c3e5a6ebffcedb423f034d2eea58eb498d9 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Tue, 22 Jul 2025 10:44:54 -0400 Subject: [PATCH 096/106] Add some comments to OutHDF5 about the global reduction --- src/OutHDF5.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 2f55672c5..4cd051bd0 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -794,9 +794,14 @@ void OutHDF5::checkParticleMasses() p = c->get_particles(&number); } - MPI_Allreduce(MPI_IN_PLACE, &minMass, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, &maxMass, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - + // Get the min and max mass for all processes + MPI_Allreduce(MPI_IN_PLACE, &minMass, 1, MPI_DOUBLE, MPI_MIN, + MPI_COMM_WORLD); + MPI_Allreduce(MPI_IN_PLACE, &maxMass, 1, MPI_DOUBLE, MPI_MAX, + MPI_COMM_WORLD); + + // All processes generate masses and multim from the globally + // reduced values if ( (maxMass - minMass)/maxMass < 1.0e-12) { masses.push_back(maxMass); multim.push_back(true); From 2dd4654804c9c1dfb037a36dc9b69262915e1acf Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 16 Aug 2025 13:19:18 -0400 Subject: [PATCH 097/106] Allow non-Python based cache construction - Replace old 'cylcache' with a version uses the pyEXP constructor - Checks for the Python3 runtime - Disable pybind11 dependence if not found - CMake will not configure if Python3 is not available with pyEXP requested --- CMakeLists.txt | 8 +- config_cmake.h_in | 3 + exputil/DiskDensityFunc.cc | 22 ++ include/DiskDensityFunc.H | 4 + utils/ICs/CMakeLists.txt | 3 +- utils/ICs/cylcache.cc | 635 +++++-------------------------------- 6 files changed, 112 insertions(+), 563 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dacf8e78..fdd951ef2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -268,7 +268,13 @@ endif() # try to find pybind11 and build wrapper python module find_package(Python3 COMPONENTS Interpreter Development) message(STATUS "python3 include dirs: ${Python3_INCLUDE_DIRS}") - +if(Python3_FOUND) + set(HAVE_PYTHON3 TRUE) +else() + if(ENABLE_PYEXP) + message(FATAL_ERROR "You asked for pyEXP but I cannot find a Python3 environment. Please make Python3 available or disable pyEXP. CMake will exit." ) + endif() +endif() # Force installation of the yaml-cpp libraries install(TARGETS yaml-cpp DESTINATION lib) diff --git a/config_cmake.h_in b/config_cmake.h_in index d1cf00db3..6f45119a0 100644 --- a/config_cmake.h_in +++ b/config_cmake.h_in @@ -19,6 +19,9 @@ /* Defined if you have HDF5 support */ #cmakedefine HAVE_HDF5 @HAVE_HDF5@ +/* Defined if Python3 runtime exists */ +#cmakedefine HAVE_PYTHON3 1 + /* Define to 1 if you have the `cuda' library. */ #cmakedefine HAVE_LIBCUDA 1 diff --git a/exputil/DiskDensityFunc.cc b/exputil/DiskDensityFunc.cc index 2c6f00bc5..1437abb17 100644 --- a/exputil/DiskDensityFunc.cc +++ b/exputil/DiskDensityFunc.cc @@ -1,5 +1,7 @@ #include +#ifdef HAVE_PYTHON3 + namespace py = pybind11; DiskDensityFunc::DiskDensityFunc(const std::string& modulename, @@ -29,3 +31,23 @@ double DiskDensityFunc::operator() (double R, double z, double phi) { return disk_density(R, z, phi).cast(); } + +#else + +DiskDensityFunc::DiskDensityFunc(const std::string& modulename, + const std::string& funcname) + : funcname(funcname) +{ + throw std::runtime_error("DiskDensityFunc: you environoment does not have Python3 support. Use a built-in density target or install Python3 and recompile"); +} + +DiskDensityFunc::~DiskDensityFunc() +{ +} + +double DiskDensityFunc::operator() (double R, double z, double phi) +{ + return 0.0; +} + +#endif diff --git a/include/DiskDensityFunc.H b/include/DiskDensityFunc.H index 0469214cb..e0bec29cf 100644 --- a/include/DiskDensityFunc.H +++ b/include/DiskDensityFunc.H @@ -3,6 +3,8 @@ #include +#include + #include #include @@ -35,8 +37,10 @@ DiskDensityFunc { private: +#ifdef HAVE_PYTHON3 //! The disk-density function pybind11::function disk_density; +#endif //! Interpreter started? bool started = false; diff --git a/utils/ICs/CMakeLists.txt b/utils/ICs/CMakeLists.txt index 0c89b9942..51ea88859 100644 --- a/utils/ICs/CMakeLists.txt +++ b/utils/ICs/CMakeLists.txt @@ -5,7 +5,7 @@ set(bin_PROGRAMS shrinkics gensph gendisk gendisk2d gsphere pstmod cubeics zangics slabics) set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil - ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} + expui ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${HDF5_CXX_LIBRARIES}) if(ENABLE_CUDA) @@ -26,6 +26,7 @@ set(common_INCLUDE $ $ $ + $ $ $ ${CMAKE_BINARY_DIR} ${DEP_INC} diff --git a/utils/ICs/cylcache.cc b/utils/ICs/cylcache.cc index e23a3cc65..ff5715bbd 100644 --- a/utils/ICs/cylcache.cc +++ b/utils/ICs/cylcache.cc @@ -1,19 +1,9 @@ -/* - Generates a cylindrical basis cache file -*/ +// Generates a cylindrical basis cache file using expui #include #include #include #include -#include -#include -#include -#include -#include -#include - -#include #include #ifdef HAVE_OMP_H @@ -23,403 +13,60 @@ // EXP classes // #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include // Command-line parsing #include // Ini-style config -#include - -#define M_SQRT1_3 (0.5773502691896257645091487) - - // For debugging -#ifdef DEBUG -#include -#include - -//=========================================== -// Handlers defined in exputil/stack.cc -//=========================================== - -extern void mpi_print_trace(const string& routine, const string& msg, - const char *file, int line); - -extern void mpi_gdb_print_trace(int sig); - -extern void mpi_gdb_wait_trace(int sig); - -//=========================================== -// A signal handler to trap invalid FP only -//=========================================== - -void set_fpu_invalid_handler(void) -{ - // Flag invalid FP results only, such as 0/0 or infinity - infinity - // or sqrt(-1). - // - feenableexcept(FE_INVALID); - // - // Print enabled flags to root node - // - if (myid==0) { - const std::list> flags = - { {FE_DIVBYZERO, "divide-by-zero"}, - {FE_INEXACT, "inexact"}, - {FE_INVALID, "invalid"}, - {FE_OVERFLOW, "overflow"}, - {FE_UNDERFLOW, "underflow"} }; - - int _flags = fegetexcept(); - std::cout << "Enabled FE flags: <"; - for (auto v : flags) { - if (v.first & _flags) std::cout << v.second << ' '; - } - std::cout << "\b>" << std::endl; - } - signal(SIGFPE, mpi_gdb_print_trace); -} - -//=========================================== -// A signal handler to produce a traceback -//=========================================== - -void set_fpu_trace_handler(void) -{ - // Flag all FP errors except inexact - // - // fedisableexcept(FE_ALL_EXCEPT & ~FE_INEXACT); - - // Flag invalid FP results only, such as 0/0 or infinity - infinity - // or sqrt(-1). - // - feenableexcept(FE_INVALID); - // - // Print enabled flags to root node - // - if (myid==0) { - const std::list> flags = - { {FE_DIVBYZERO, "divide-by-zero"}, - {FE_INEXACT, "inexact"}, - {FE_INVALID, "invalid"}, - {FE_OVERFLOW, "overflow"}, - {FE_UNDERFLOW, "underflow"} }; - - int _flags = fegetexcept(); - std::cout << "Enabled FE flags: <"; - for (auto v : flags) { - if (v.first & _flags) std::cout << v.second << ' '; - } - std::cout << "\b>" << std::endl; - } - signal(SIGFPE, mpi_gdb_print_trace); -} - -//=========================================== -// A signal handler to produce stop and wait -//=========================================== - -void set_fpu_gdb_handler(void) -{ - // Flag all FP errors except inexact - // - // fedisableexcept(FE_ALL_EXCEPT & ~FE_INEXACT); - - // Flag invalid FP results only, such as 0/0 or infinity - infinity - // or sqrt(-1). - // - feenableexcept(FE_INVALID); - // - // Print enabled flags to root node - // - if (myid==0) { - const std::list> flags = - { {FE_DIVBYZERO, "divide-by-zero"}, - {FE_INEXACT, "inexact"}, - {FE_INVALID, "invalid"}, - {FE_OVERFLOW, "overflow"}, - {FE_UNDERFLOW, "underflow"} }; - - int _flags = fegetexcept(); - std::cout << "Enabled FE flags: <"; - for (auto v : flags) { - if (v.first & _flags) std::cout << v.second << ' '; - } - std::cout << "\b>" << std::endl; - } - signal(SIGFPE, mpi_gdb_wait_trace); -} - -#endif - - // Local headers -#include - - -// Global variables -// -enum DiskType { constant, gaussian, mn, exponential, doubleexpon, diskbulge, python }; - -std::map dtlookup = - { {"constant", DiskType::constant}, - {"gaussian", DiskType::gaussian}, - {"mn", DiskType::mn}, - {"exponential", DiskType::exponential}, - {"doubleexpon", DiskType::doubleexpon}, - {"diskbulge", DiskType::diskbulge}, - {"python", DiskType::python} - }; - -DiskType DTYPE; -std::string pyname; -double ASCALE; -double ASHIFT; -double HSCALE; -double HERNA; -double Mfac; -bool sech2; -double RTRUNC = 1.0; -double RWIDTH = 0.0; -double ARATIO = 1.0; -double HRATIO = 1.0; -double DWEIGHT = 1.0; - -#include - -static std::shared_ptr pydens; - -double DiskDens(double R, double z, double phi) -{ - double ans = 0.0; - - switch (DTYPE) { - - case DiskType::constant: - if (R < ASCALE && fabs(z) < HSCALE) - ans = 1.0/(2.0*HSCALE*M_PI*ASCALE*ASCALE); - break; - - case DiskType::gaussian: - if (fabs(z) < HSCALE) - ans = 1.0/(2.0*HSCALE*2.0*M_PI*ASCALE*ASCALE)* - exp(-R*R/(2.0*ASCALE*ASCALE)); - break; - - case DiskType::mn: - { - double Z2 = z*z + HSCALE*HSCALE; - double Z = sqrt(Z2); - double Q2 = (ASCALE + Z)*(ASCALE + Z); - ans = 0.25*HSCALE*HSCALE/M_PI*(ASCALE*R*R + (ASCALE + 3.0*Z)*Q2)/( pow(R*R + Q2, 2.5) * Z*Z2 ); - } - break; - - case DiskType::doubleexpon: - { - double a1 = ASCALE; - double a2 = ASCALE*ARATIO; - double h1 = sech2 ? 0.5*HSCALE : HSCALE; - double h2 = h1*HRATIO; - double w1 = 1.0/(1.0+DWEIGHT); - double w2 = DWEIGHT/(1.0+DWEIGHT); - - double f1 = cosh(z/h1); - double f2 = cosh(z/h2); - - ans = - w1*exp(-R/a1)/(4.0*M_PI*a1*a1*h1*f1*f1) + - w2*exp(-R/a2)/(4.0*M_PI*a2*a2*h2*f2*f2) ; - } - break; - case DiskType::diskbulge: - { - double h = sech2 ? 0.5*HSCALE : HSCALE; - double f = cosh(z/h); - double rr = pow(pow(R, 2) + pow(z,2), 0.5); - double w1 = Mfac; - double w2 = (1-Mfac); - double as = HERNA; - - ans = w1*exp(-R/ASCALE)/(4.0*M_PI*ASCALE*ASCALE*h*f*f) + - w2*pow(as, 4)/(2.0*M_PI*rr)*pow(rr+as,-3.0) ; - } - break; - case DiskType::python: - { - if (not pydens) pydens = std::make_shared(pyname); - ans = (*pydens)(R, z, phi); - } - break; - case DiskType::exponential: - default: - { - double h = sech2 ? 0.5*HSCALE : HSCALE; - double f = cosh(z/h); - ans = exp(-R/ASCALE)/(4.0*M_PI*ASCALE*ASCALE*h*f*f); - } - break; - } - - if (RWIDTH>0.0) ans *= erf((RTRUNC-R)/RWIDTH); - - return ans; -} - -double dcond(double R, double z, double phi, int M) -{ - // - // No shift for M==0 - // - if (M==0) return DiskDens(R, z, phi); - - // - // Fold into [-PI/M, PI/M] for M>=1 - // - double dmult = M_PI/M, phiS; - if (phi>M_PI) - phiS = phi + dmult*(int)((2.0*M_PI - phi)/dmult); - else - phiS = phi - dmult*(int)(phi/dmult); - - // - // Apply a shift along the x-axis - // - double x = R*cos(phiS) - ASHIFT*ASCALE; - double y = R*sin(phiS); - - return DiskDens(sqrt(x*x + y*y), z, atan2(y, x)); -} - int main(int ac, char **av) { - //==================== - // Inialize MPI stuff - //==================== + // Parameters + // + int nthrds = 1; + std::string config; + + // Default/example YAML file for disk + // + std::string disk_config = + "id : cylinder\n" + "parameters :\n" + " acyl : 1.0\n" + " hcyl : 0.1\n" + " lmaxfid : 48\n" + " nmaxfid : 48\n" + " mmax : 10\n" + " nmax : 32\n" + " ncylodd : 6\n" + " ncylnx : 256\n" + " ncylny : 128\n" + " ncylr : 2000\n" + " rnum : 200\n" + " pnum : 1\n" + " tnum : 80\n" + " rcylmin : 0.001\n" + " rcylmax : 20\n" + " ashift : 0\n" + " logr : true\n" + " expcond : true\n" + " deproject : true\n" + " cachefile : .eof.cache.file_new\n" + " ignore : true\n" + " vflag : 16\n" + ; + // Inialize MPI stuff + // local_init_mpi(ac, av); - //==================== - // Begin opt parsing - //==================== - - double RCYLMIN; - double RCYLMAX; - double RFACTOR; - int RNUM; - int PNUM; - int TNUM; - int VFLAG; - bool expcond; - bool LOGR; - int NUMR; - int CMAPR; - int CMAPZ; - int CMTYPE; - int NMAXFID; - int LMAXFID; - int MMAX; - int NUMX; - int NUMY; - int NOUT; - int NMAX; - int NCYLODD; - double PPower; - std::string cachefile; - std::string config; - std::string dtype; - std::string dmodel; - std::string mtype; - std::string ctype; - - const std::string mesg("Generates an EmpCylSL cache\n"); + const std::string mesg("Generates an EmpCylSL cache from a YAML configuration file\n"); cxxopts::Options options(av[0], mesg); options.add_options() ("h,help", "Print this help message") - ("T,template", "Write template options file with current and all default values") - ("c,config", "Parameter configuration file", - cxxopts::value(config)) - ("deproject", "The EmpCylSL deprojection from specified disk model (EXP or MN)", - cxxopts::value(dmodel)->default_value("EXP")) - ("cachefile", "The cache file for the cylindrical basis", - cxxopts::value(cachefile)->default_value(".eof.cache.file")) - ("ctype", "DiskHalo radial coordinate scaling type (one of: Linear, Log, Rat)", - cxxopts::value(ctype)->default_value("Log")) - ("LMAXFID", "Maximum angular order for spherical basis in adaptive construction of the cylindrical basis", - cxxopts::value(LMAXFID)->default_value("32")) - ("NMAXFID", "Maximum radial order for the spherical basis in adapative construction of the cylindrical basis", - cxxopts::value(NMAXFID)->default_value("32")) - ("MMAX", "Maximum azimuthal order for the cylindrical basis", - cxxopts::value(MMAX)->default_value("6")) - ("NUMX", "Size of the (mapped) cylindrical radial grid", - cxxopts::value(NUMX)->default_value("256")) - ("NUMY", "Size of the (mapped) cylindrical vertical grid", - cxxopts::value(NUMY)->default_value("128")) - ("dtype", "Spherical model type for adpative basis creation", - cxxopts::value(dtype)->default_value("exponential")) - ("NCYLODD", "Number of vertically odd basis functions per harmonic order", - cxxopts::value(NCYLODD)->default_value("6")) - ("NMAX", "Total number of basis functions per harmonic order", - cxxopts::value(NMAX)->default_value("20")) - ("VFLAG", "Diagnostic flag for EmpCylSL", - cxxopts::value(VFLAG)->default_value("31")) - ("expcond", "Use analytic target density rather than particle distribution", - cxxopts::value(expcond)->default_value("true")) - ("LOGR", "Logarithmic scaling for model table in EmpCylSL", - cxxopts::value(expcond)->default_value("true")) - ("sech2", "if true, use hcyl as sech^2(z/(2*hcyl))", - cxxopts::value(sech2)->default_value("false")) - ("RCYLMIN", "Minimum disk radius for EmpCylSL", - cxxopts::value(RCYLMIN)->default_value("0.001")) - ("RCYLMAX", "Maximum disk radius for EmpCylSL", - cxxopts::value(RCYLMAX)->default_value("20.0")) - ("ASCALE", "Radial scale length for disk basis construction", - cxxopts::value(ASCALE)->default_value("1.0")) - ("ASHIFT", "Fraction of scale length for shift in conditioning function", - cxxopts::value(ASHIFT)->default_value("0.0")) - ("HSCALE", "Vertical scale length for disk basis construction", - cxxopts::value(HSCALE)->default_value("0.1")) - ("ARATIO", "Radial scale length ratio for disk basis construction with doubleexpon", - cxxopts::value(ARATIO)->default_value("1.0")) - ("HRATIO", "Vertical scale height ratio for disk basis construction with doubleexpon", - cxxopts::value(HRATIO)->default_value("1.0")) - ("NUMR", "Size of radial grid", - cxxopts::value(NUMR)->default_value("2000")) - ("PPOW", "Power-law index for power-law disk profile", - cxxopts::value(PPower)->default_value("4.0")) - ("DWEIGHT", "Ratio of second disk relative to the first disk for disk basis construction with double-exponential", - cxxopts::value(DWEIGHT)->default_value("1.0")) - ("Mfac", "Mass fraction of the disk for diskbulge model, sets disk/bulge mass (e.g. 0.75 -> 75% of mass in disk, 25% of mass in bulge)", - cxxopts::value(Mfac)->default_value("1.0")) - ("HERNA", "Hernquist scale a parameter in diskbulge model", - cxxopts::value(HERNA)->default_value("0.10")) - ("RTRUNC", "Maximum disk radius for erf truncation of EOF conditioning density", - cxxopts::value(RTRUNC)->default_value("0.1")) - ("RWIDTH", "Width for erf truncation for EOF conditioning density (ignored if zero)", - cxxopts::value(RWIDTH)->default_value("0.0")) - ("RFACTOR", "Disk radial scaling factor for spherical deprojection model", - cxxopts::value(RFACTOR)->default_value("1.0")) - ("RNUM", "Number of radial knots for EmpCylSL basis construction quadrature", - cxxopts::value(RNUM)->default_value("200")) - ("PNUM", "Number of azimthal knots for EmpCylSL basis construction quadrature", - cxxopts::value(PNUM)->default_value("1")) - ("TNUM", "Number of cos(theta) knots for EmpCylSL basis construction quadrature", - cxxopts::value(TNUM)->default_value("80")) - ("CMAPR", "Radial coordinate mapping type for cylindrical grid (0=none, 1=rational fct)", - cxxopts::value(CMAPR)->default_value("1")) - ("CMAPZ", "Vertical coordinate mapping type for cylindrical grid (0=none, 1=rational fct)", - cxxopts::value(CMAPZ)->default_value("1")) - ("pyname", "The name of the Python module supplying the disk_density function", - cxxopts::value(pyname)) + ("T,template", "Write template YAML config file") + ("c,config", "Read a configuration file", cxxopts::value(config)) + ("t,threads", "Number of OpenMP threads", cxxopts::value(nthrds)) ; cxxopts::ParseResult vm; @@ -435,29 +82,26 @@ main(int ac, char **av) // Write YAML template config file and exit // if (vm.count("template")) { - NOUT = NMAX;//std::min(NOUT, NMAX); - // Write template file // - if (myid==0) SaveConfig(vm, options, "template.yaml"); + if (myid==0) { + std::ofstream out("template.yaml"); + if (out) out << disk_config; + std::cout << "Wrote . Please adjust to your preferences" + << std::endl; + } else { + std::cerr << "Error opening for output" << std::endl; + } MPI_Finalize(); return 0; } - // Print help message and exit // if (vm.count("help")) { if (myid == 0) { - std::cout << options.help() << std::endl << std::endl - << "Examples: " << std::endl - << "\t" << "Use parameters read from a config file in INI style" << std::endl - << "\t" << av[0] << " --config=gendisk.config" << std::endl << std::endl - << "\t" << "Generate a template YAML config file current defaults called " << std::endl - << "\t" << av[0] << " --template" << std::endl << std::endl - << "\t" << "Override a single parameter in a config file from the command line" << std::endl - << "\t" << av[0] << "--LMAX=8 --config=my.config" << std::endl << std::endl; + std::cout << options.help() << std::endl << std::endl; } MPI_Finalize(); return 0; @@ -467,78 +111,25 @@ main(int ac, char **av) // if (vm.count("config")) { try { - vm = LoadConfig(options, config); - } catch (cxxopts::OptionException& e) { - if (myid==0) std::cout << "Option error in configuration file: " + auto name = vm["config"].as(); + auto size = std::filesystem::file_size(name); + std::string content(size, '\0'); + std::ifstream in(name); + in.read(&content[0], size); + disk_config = content; + } catch (std::runtime_error& e) { + if (myid==0) std::cout << "Error reading configuration file: " << e.what() << std::endl; MPI_Finalize(); return 0; } } - // Convert mtype string to lower case - // - std::transform(mtype.begin(), mtype.end(), mtype.begin(), - [](unsigned char c){ return std::tolower(c); }); - - // Set EmpCylSL mtype. This is the spherical function used to - // generate the EOF basis. If "deproject" is set, this will be - // overriden in EmpCylSL. - // - EmpCylSL::mtype = EmpCylSL::Exponential; - if (vm.count("mtype")) { - if (mtype.compare("exponential")==0) - EmpCylSL::mtype = EmpCylSL::Exponential; - else if (mtype.compare("gaussian")==0) - EmpCylSL::mtype = EmpCylSL::Gaussian; - else if (mtype.compare("plummer")==0) - EmpCylSL::mtype = EmpCylSL::Plummer; - else if (mtype.compare("power")==0) { - EmpCylSL::mtype = EmpCylSL::Power; - EmpCylSL::PPOW = PPower; - } else { - if (myid==0) std::cout << "No EmpCylSL EmpModel named <" - << mtype << ">, valid types are: " - << "Exponential, Gaussian, Plummer, Power " - << "(not case sensitive)" << std::endl; - MPI_Finalize(); - return 0; - } - } - - // Convert dtype string to lower case + // Explicit OpenMP control; otherwise, fall back to the environment + // setting OMP_NUM_THREADS. // - std::transform(dtype.begin(), dtype.end(), dtype.begin(), - [](unsigned char c){ return std::tolower(c); }); - - - // Set DiskType. This is the functional form for the disk used to - // condition the basis. - // - try { // Check for map entry, will through if the - DTYPE = dtlookup.at(dtype); // key is not in the map. - - if (myid==0) // Report DiskType - std::cout << "---- DiskType is <" << dtype << ">" << std::endl; - } - catch (const std::out_of_range& err) { - if (myid==0) { - std::cout << "DiskType error in configuraton file" << std::endl; - std::cout << "Valid options are: "; - for (auto v : dtlookup) std::cout << v.first << " "; - std::cout << std::endl; - } - MPI_Finalize(); - return 0; - } - - - //==================== - // OpenMP control - //==================== - #ifdef HAVE_OMP_H - omp_set_num_threads(nthrds); + if (vm.count("threads")) omp_set_num_threads(nthrds); #pragma omp parallel if (myid==0) { int numthrd = omp_get_num_threads(); @@ -546,104 +137,26 @@ main(int ac, char **av) } #endif - - //==================== - // Okay, now begin ... - //==================== - -#ifdef DEBUG // For gdb . . . - sleep(20); - // set_fpu_handler(); // Make gdb trap FPU exceptions - set_fpu_gdb_handler(); // Make gdb trap FPU exceptions -#endif - - - - //===========================Cylindrical expansion=========================== - - - EmpCylSL::RMIN = RCYLMIN; - EmpCylSL::RMAX = RCYLMAX; - EmpCylSL::NUMX = NUMX; - EmpCylSL::NUMY = NUMY; - EmpCylSL::NUMR = NUMR; - EmpCylSL::NOUT = NMAX; - EmpCylSL::CMAPR = CMAPR; - EmpCylSL::CMAPZ = CMAPZ; - EmpCylSL::VFLAG = VFLAG; - EmpCylSL::logarithmic = LOGR; - - // Create expansion only if needed . . . - std::shared_ptr expandd; - - // - expandd = std::make_shared(NMAXFID, LMAXFID, MMAX, NMAX, ASCALE, HSCALE, NCYLODD, cachefile); - -#ifdef DEBUG - std::cout << "Process " << myid << ": " - << " rmin=" << EmpCylSL::RMIN - << " rmax=" << EmpCylSL::RMAX - << " a=" << ASCALE - << " h=" << HSCALE - << " sech2=" << sech2 - << " as=" << HERNA - << " Mfac=" << Mfac - << " nmax2=" << NMAXFID - << " lmax2=" << LMAXFID - << " mmax=" << MMAX - << " nordz=" << NMAX - << " noddz=" << NCYLODD - << std::endl << std::flush; -#endif - - - // Use these user models to deproject for the EOF spherical basis + // Begin calculation and start stopwatch // - if (vm.count("deproject")) { - // The scale in EmpCylSL is assumed to be 1 so we compute the - // height relative to the length - // - //double H = HSCALE/ASCALE; - double H = sech2 ? 0.5*HSCALE/ASCALE : HSCALE/ASCALE; - - // The model instance (you can add others in DiskModels.H). - // It's MN or Exponential if not MN. - // - EmpCylSL::AxiDiskPtr model; - - if (dmodel.compare("MN")==0) // Miyamoto-Nagai - model = std::make_shared(1.0, H); - else // Default to exponential - model = std::make_shared(1.0, H); - - if (RWIDTH>0.0) { - model = std::make_shared(RTRUNC/ASCALE, - RWIDTH/ASCALE, - model); - if (myid==0) - std::cout << "Made truncated model with R=" << RTRUNC/ASCALE - << " and W=" << RWIDTH/ASCALE << std::endl; - } + auto t_start = MPI_Wtime(); - expandd->create_deprojection(H, RFACTOR, NUMR, RNUM, model); - } - - // Regenerate EOF from analytic density + // Construct the basis instance // - expandd->generate_eof(RNUM, PNUM, TNUM, dcond); + auto disk_basis = BasisClasses::Basis::factory_string(disk_config); + + MPI_Barrier(MPI_COMM_WORLD); - // Basis orthgonality check + // DONE // - if (vm.count("ortho")) { - std::ofstream out(runtag + ".ortho_check"); - expandd->ortho_check(out); - } + if (myid==0) + std::cout << std::string(60, '=') << std::endl + << "Calculation finished in " << std::setprecision(3) + << MPI_Wtime() - t_start << " seconds" << std::endl + << std::string(60, '=') << std::endl; - //=========================================================================== // Shutdown MPI - //=========================================================================== - - MPI_Barrier(MPI_COMM_WORLD); + // MPI_Finalize(); return 0; From 0b8ee25b5a4d1ae5c2bc0682ba3f5e5ac6acf496 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 16 Aug 2025 13:32:14 -0400 Subject: [PATCH 098/106] Remove the dot in the default cachefile name; decrease the verbosity to stdout --- utils/ICs/cylcache.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/ICs/cylcache.cc b/utils/ICs/cylcache.cc index ff5715bbd..5e2d7cc1f 100644 --- a/utils/ICs/cylcache.cc +++ b/utils/ICs/cylcache.cc @@ -49,9 +49,9 @@ main(int ac, char **av) " logr : true\n" " expcond : true\n" " deproject : true\n" - " cachefile : .eof.cache.file_new\n" + " cachefile : eof.cache.file_new\n" " ignore : true\n" - " vflag : 16\n" + " vflag : 5\n" ; // Inialize MPI stuff From c0c0b0948c35630e755ef78020c9a3010b417076 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Sat, 16 Aug 2025 14:00:37 -0400 Subject: [PATCH 099/106] Removed deprecated parameters from the default YAML config --- utils/ICs/cylcache.cc | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/utils/ICs/cylcache.cc b/utils/ICs/cylcache.cc index 5e2d7cc1f..56f44af20 100644 --- a/utils/ICs/cylcache.cc +++ b/utils/ICs/cylcache.cc @@ -49,8 +49,7 @@ main(int ac, char **av) " logr : true\n" " expcond : true\n" " deproject : true\n" - " cachefile : eof.cache.file_new\n" - " ignore : true\n" + " cachename : eof.cache.file_new\n" " vflag : 5\n" ; @@ -82,17 +81,16 @@ main(int ac, char **av) // Write YAML template config file and exit // if (vm.count("template")) { - // Write template file - // if (myid==0) { std::ofstream out("template.yaml"); - if (out) out << disk_config; - std::cout << "Wrote . Please adjust to your preferences" - << std::endl; - } else { - std::cerr << "Error opening for output" << std::endl; + if (out) { + out << disk_config; + std::cout << "Wrote . Please adjust to your preferences" + << std::endl; + } else { + std::cerr << "Error opening for output" << std::endl; + } } - MPI_Finalize(); return 0; } From 73212ccc5c2415ed7d26792a0d0d45d041300342 Mon Sep 17 00:00:00 2001 From: Michael Petersen Date: Mon, 29 Sep 2025 13:39:13 +0100 Subject: [PATCH 100/106] Update utils/PhaseSpace/diffpsp.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- utils/PhaseSpace/diffpsp.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/PhaseSpace/diffpsp.cc b/utils/PhaseSpace/diffpsp.cc index 7a52b03bd..941e34e71 100755 --- a/utils/PhaseSpace/diffpsp.cc +++ b/utils/PhaseSpace/diffpsp.cc @@ -576,8 +576,8 @@ main(int argc, char **argv) << "-- I2max: " << I2max << std::endl << std::string(40, '-') << std::endl; } else { - if (I1min>0) I1min = Emin; - if (I1max>0) I1max = Emax; + if (I1min<0) I1min = Emin; + if (I1max<0) I1max = Emax; if (I2min<0) I2min = std::max(KTOL, KMIN); if (I2max<0) I2max = std::min(1.0 - KTOL, KMAX); From 2077fff87e0271a7a03df486986b153a29f91013 Mon Sep 17 00:00:00 2001 From: Michael Petersen Date: Mon, 29 Sep 2025 13:40:28 +0100 Subject: [PATCH 101/106] Update expui/expMSSA.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- expui/expMSSA.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expui/expMSSA.cc b/expui/expMSSA.cc index f63f2d063..0b6b5caf4 100644 --- a/expui/expMSSA.cc +++ b/expui/expMSSA.cc @@ -1523,7 +1523,7 @@ namespace MSSA { recon.createAttribute ("ncomp", HighFive::DataSpace::From(ncomp) ).write(ncomp); recon.createAttribute("totVar", HighFive::DataSpace::From(totVar)).write(totVar); - recon.createAttribute("totPow", HighFive::DataSpace::From(totVar)).write(totPow); + recon.createAttribute("totPow", HighFive::DataSpace::From(totPow)).write(totPow); for (int n=0; n Date: Mon, 29 Sep 2025 13:42:59 +0100 Subject: [PATCH 102/106] Update utils/PhaseSpace/diffpsp.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- utils/PhaseSpace/diffpsp.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/PhaseSpace/diffpsp.cc b/utils/PhaseSpace/diffpsp.cc index 941e34e71..50bec0c20 100755 --- a/utils/PhaseSpace/diffpsp.cc +++ b/utils/PhaseSpace/diffpsp.cc @@ -633,7 +633,7 @@ main(int argc, char **argv) int npath1 = 1, npath2 = 1; if (fileType != "PSPhdf5") { npath1 = INFILE1.size(); - npath2 = INFILE1.size(); + npath2 = INFILE2.size(); } // Iterate through file list From 844b263d466a7504f9258bfc84c6bb63b3d1b3d3 Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Mon, 29 Sep 2025 12:57:18 -0400 Subject: [PATCH 103/106] Update expui/FieldGenerator.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- expui/FieldGenerator.cc | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/expui/FieldGenerator.cc b/expui/FieldGenerator.cc index a71470dda..9bf4ca921 100644 --- a/expui/FieldGenerator.cc +++ b/expui/FieldGenerator.cc @@ -984,15 +984,24 @@ namespace Field double rf = 4.0*pi/3.0*(d3 - 1.0); for (int i=0; i 0) { + for (int k=0; k<3; k++) { + vc1(i, k) /= ret[i]; + vc2(i, k) /= ret[i]; + sig += vc2(i, k) - vc1(i, k)*vc1(i, k); + } + rad[i] = exp(lrmin + del*(0.5 + i)); + ret[i] /= exp(3.0*(lrmin + del*i)) * rf; + vel[i] = sqrt(fabs(sig)); + } else { + for (int k=0; k<3; k++) { + vc1(i, k) = 0.0; + vc2(i, k) = 0.0; + } + rad[i] = exp(lrmin + del*(0.5 + i)); + ret[i] = 0.0; + vel[i] = 0.0; } - - rad[i] = exp(lrmin + del*(0.5 + i)); - ret[i] /= exp(3.0*(lrmin + del*i)) * rf; - vel[i] = sqrt(fabs(sig)); } return {rad, ret, vel}; From 21002fcc879332ad3dbb7d7b718768724f8ca67a Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 29 Sep 2025 13:29:47 -0400 Subject: [PATCH 104/106] Fix default assignments --- utils/PhaseSpace/diffpsp.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/utils/PhaseSpace/diffpsp.cc b/utils/PhaseSpace/diffpsp.cc index 5b62f1dcb..c80e7d99a 100755 --- a/utils/PhaseSpace/diffpsp.cc +++ b/utils/PhaseSpace/diffpsp.cc @@ -554,16 +554,16 @@ main(int argc, char **argv) Emax *= 1.0 + KTOL; orb.new_orbit(Emin, 1.0 - KTOL); - if (vm.count("I1min")>0) I1min = orb.get_action(0); + if (vm.count("I1min")==0) I1min = orb.get_action(0); orb.new_orbit(Emin, KTOL); - if (vm.count("I2min")>0) I2min = orb.get_action(1); + if (vm.count("I2min")==0) I2min = orb.get_action(1); orb.new_orbit(Emax, KTOL); - if (vm.count("I1max")>0) I1max = orb.get_action(0); + if (vm.count("I1max")==0) I1max = orb.get_action(0); orb.new_orbit(Emax, 1.0 - KTOL); - if (vm.count("I2max")>0) I2max = orb.get_action(1); + if (vm.count("I2max")==0) I2max = orb.get_action(1); if (myid==0) std::cout << std::endl @@ -576,10 +576,10 @@ main(int argc, char **argv) << "-- I2max: " << I2max << std::endl << std::string(40, '-') << std::endl; } else { - if (vm.count("I1min")>0) I1min = Emin; - if (vm.count("I1max")>0) I1max = Emax; - if (vm.count("I2min")>0) I2min = std::max(KTOL, KMIN); - if (vm.count("I2max")>0) I2max = std::min(1.0 - KTOL, KMAX); + if (vm.count("I1min")==0) I1min = Emin; + if (vm.count("I1max")==0) I1max = Emax; + if (vm.count("I2min")==0) I2min = std::max(KTOL, KMIN); + if (vm.count("I2max")==0) I2max = std::min(1.0 - KTOL, KMAX); if (myid==0) std::cout << std::endl From e59180da61cb5d5854e1eff5c56243e91209f58a Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 29 Sep 2025 13:55:54 -0400 Subject: [PATCH 105/106] Fix fencepost error in particle component index --- src/OutHDF5.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/OutHDF5.cc b/src/OutHDF5.cc index 4cd051bd0..b744d9c43 100644 --- a/src/OutHDF5.cc +++ b/src/OutHDF5.cc @@ -546,7 +546,7 @@ void OutHDF5::RunGadget4(const std::string& path) #endif std::ostringstream sout; - sout << "PartType" << count++; + sout << "PartType" << count; auto pgroup = file->createGroup(sout.str()); @@ -554,6 +554,8 @@ void OutHDF5::RunGadget4(const std::string& path) c->write_HDF5(pgroup, multim[count], ids); else c->write_HDF5(pgroup, multim[count], ids); + + count++; } } @@ -759,7 +761,7 @@ void OutHDF5::RunPSP(const std::string& path) #endif std::ostringstream sout; - sout << "PartType" << count++; + sout << "PartType" << count; H5::Group group = file.createGroup(sout.str()); @@ -767,6 +769,8 @@ void OutHDF5::RunPSP(const std::string& path) c->write_H5(group); else c->write_H5(group); + + count++; } } catch (H5::Exception& error) { throw std::runtime_error(std::string("OutHDF5: error writing HDF5 file ") + error.getDetailMsg()); @@ -810,4 +814,5 @@ void OutHDF5::checkParticleMasses() multim.push_back(false); } } + // END: component loop } From 90aec55081cba58d41eb3d64e1e17fd983a282b5 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Mon, 29 Sep 2025 14:01:35 -0400 Subject: [PATCH 106/106] Make initial small DF offset value a constant --- exputil/massmodel_dist.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/exputil/massmodel_dist.cc b/exputil/massmodel_dist.cc index 618a4bd8e..e6877b48a 100644 --- a/exputil/massmodel_dist.cc +++ b/exputil/massmodel_dist.cc @@ -154,8 +154,10 @@ void SphericalModelTable::setup_df(int NUM, double RA) Qmin = get_pot(pot.x[0]); dQ = (Qmax - Qmin)/(double)(dfc.num-1); - // foffset = -std::numeric_limits::max(); - foffset = -1.0e42; + // The small value is not critical + const double FOFFSET0 = -1.0e-42; + foffset = FOFFSET0; + dfc.Q[dfc.num-1] = Qmax; dfc.ffQ[dfc.num-1] = 0.0; fac = 1.0/(sqrt(8.0)*M_PI*M_PI);