Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
cmake_minimum_required(VERSION 3.21) # Needed for CUDA, MPI, and CTest features
cmake_minimum_required(VERSION 3.25) # Needed for CUDA, MPI, and CTest features

project(
EXP
Expand Down Expand Up @@ -122,6 +122,9 @@ if(OpenMP_FOUND)
OPTION (USE_OpenMP "Use OpenMP" ON)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
if(ENABLE_CUDA)
set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler='${OpenMP_CXX_FLAGS}'")
endif()
endif()
# Slurm support
if(SLURM_FOUND)
Expand Down
8 changes: 6 additions & 2 deletions expui/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>


if(ENABLE_CUDA)
list(APPEND common_INCLUDE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(SLURM_FOUND)
Expand Down
13 changes: 9 additions & 4 deletions exputil/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,18 @@ set(common_INCLUDE_DIRS $<INSTALL_INTERFACE:include>
${DEP_INC} ${EIGEN3_INCLUDE_DIR} ${HDF5_INCLUDE_DIRS}
${FFTW_INCLUDE_DIRS})

set(common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
set(common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
yaml-cpp ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}
${FFTW_DOUBLE_LIB})

if(ENABLE_CUDA)
list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
set_source_files_properties(${BIORTH_SRC} ${PARTICLE_SRC} PROPERTIES LANGUAGE CUDA)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(ENABLE_XDR AND TIRPC_FOUND)
Expand All @@ -70,7 +75,7 @@ endif()
add_library(exputil ${exputil_SOURCES})
set_target_properties(exputil PROPERTIES OUTPUT_NAME exputil)
target_include_directories(exputil PUBLIC ${common_INCLUDE_DIRS})
target_link_libraries(exputil PUBLIC ${common_LINKLIBS})
target_link_libraries(exputil PUBLIC ${common_LINKLIB})

install(TARGETS exputil DESTINATION lib)

Expand Down
7 changes: 6 additions & 1 deletion pyEXP/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>
${HDF5_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR})

if(ENABLE_CUDA)
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(SLURM_FOUND)
Expand Down
31 changes: 19 additions & 12 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,12 +1,3 @@

set(CUDA_SRC)
if (ENABLE_CUDA)
list(APPEND CUDA_SRC cudaPolarBasis.cu cudaSphericalBasis.cu
cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc
cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu
cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu)
endif()

set(exp_SOURCES Basis.cc Bessel.cc Component.cc
Cube.cc Cylinder.cc ExternalCollection.cc
ExternalForce.cc Orient.cc PotAccel.cc ScatterMFP.cc
Expand All @@ -19,9 +10,16 @@ set(exp_SOURCES Basis.cc Bessel.cc Component.cc
OutMulti.cc OutRelaxation.cc OrbTrace.cc OutDiag.cc OutLog.cc
OutVel.cc OutCoef.cc multistep.cc parse.cc SlabSL.cc step.cc
tidalField.cc ultra.cc ultrasphere.cc MPL.cc OutFrac.cc OutCalbr.cc
ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc ${CUDA_SRC}
ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc
CenterFile.cc PolarBasis.cc FlatDisk.cc signals.cc)

if (ENABLE_CUDA)
list(APPEND exp_SOURCES cudaPolarBasis.cu cudaSphericalBasis.cu
cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc
cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu
cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu)
endif()

set(common_INCLUDE_DIRS
$<INSTALL_INTERFACE:include>
$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include/>
Expand All @@ -39,8 +37,13 @@ if(PNG_FOUND)
endif()

if(ENABLE_CUDA)
list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
set_source_files_properties(${exp_SOURCES} PROPERTIES LANGUAGE CUDA)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(SLURM_FOUND)
Expand All @@ -64,5 +67,9 @@ add_executable(exp expand.cc)
target_include_directories(exp PUBLIC ${common_INCLUDE_DIRS})
target_link_libraries(exp PUBLIC ${common_LINKLIB} EXPlib)

if (ENABLE_CUDA)
set_target_properties(exp PROPERTIES LINKER_LANGUAGE CUDA)
endif ()

install(TARGETS EXPlib DESTINATION lib)
install(TARGETS exp DESTINATION bin)
11 changes: 1 addition & 10 deletions src/Cylinder.H
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#include <cudaParticle.cuH>
#include <cudaMappingConstants.cuH>
#endif
#include <NVTX.H>

#include <CoefContainer.H>

Expand Down Expand Up @@ -155,15 +154,7 @@ private:
a complete set of coefficients for force evaluation at an
intermediate time step
*/
void compute_multistep_coefficients()
{
if (play_back and not play_cnew) return;

nvTracerPtr tPtr;
if (cuda_prof)
tPtr = nvTracerPtr(new nvTracer("Cylinder::compute_multistep_coefficients"));
ortho->compute_multistep_coefficients(mfirst[mstep]);
}
void compute_multistep_coefficients();

//! Reset used particle counter and mass registers
virtual void multistep_reset();
Expand Down
11 changes: 11 additions & 0 deletions src/Cylinder.cc
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a nice maintenance change, forced by the new cuda compile strategy, but I think it also makes the codebase simpler.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right. The goal here was to get the implementation out of the header where it required NVTX-specific structures.

Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
#include <MixtureBasis.H>
#include <Timer.H>
#include <exputils.H>
#include <NVTX.H>

Timer timer_debug;

Expand Down Expand Up @@ -1659,6 +1660,16 @@ void Cylinder::multistep_update(int from, int to, Component* c, int i, int id)
#endif
}

void Cylinder::compute_multistep_coefficients()
{
if (play_back and not play_cnew) return;

nvTracerPtr tPtr;
if (cuda_prof)
tPtr = nvTracerPtr(new nvTracer("Cylinder::compute_multistep_coefficients"));
ortho->compute_multistep_coefficients(mfirst[mstep]);
}


void Cylinder::multistep_reset()
{
Expand Down
1 change: 1 addition & 0 deletions src/ExternalForce.H
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this is still for testing?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like cruft to me.

Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public:
//! Finish and clean-up (caching data necessary for restart)
virtual void finish() {}

// #if HAVE_LIBCUDA==1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
// #if HAVE_LIBCUDA==1

#if HAVE_LIBCUDA==1
//! Copy particles from device for non-cuda forces
void getParticlesCuda(Component *c);
Expand Down
4 changes: 4 additions & 0 deletions src/NVTX.H
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@

#if HAVE_LIBCUDA==1

#if __CUDACC_VER_MAJOR__ < 12
#include <nvToolsExt.h>
#else
#include <nvtx3/nvToolsExt.h>
#endif

class nvTracer {
private:
Expand Down
2 changes: 1 addition & 1 deletion src/NVTX.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#include <NVTX.H>
#include "NVTX.H"

#if HAVE_LIBCUDA==1

Expand Down
2 changes: 0 additions & 2 deletions src/Orient.H
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we planning on stylistically removing this everywhere? Fine if so and I'll start when I touch any other code.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not really stylistic. In most compilers, using the "" first checks your local directory, and if it doesn't find a match then moves on to check the system paths. Using <> starts the search with system headers. NVTX.H should always be local. So I would say: use "" when we really mean local directory.

Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
// This may look like C code, but it is really -*- C++ -*-

#ifndef _Orient_H
#define _Orient_H

Expand Down
1 change: 1 addition & 0 deletions src/cudaCylinder.cu
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Clean fix here, thanks!

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that's old cruft from emacs.

Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <Component.H>
#include <Cylinder.H>
#include <cudaReduce.cuH>
#include <float.h>
#include "expand.H"

// Define for debugging
Expand Down
1 change: 1 addition & 0 deletions src/cudaPolarBasis.cu
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <Component.H>
#include <PolarBasis.H>
#include <cudaReduce.cuH>
#include <float.h>
#include "expand.H"

// Define for debugging
Expand Down
4 changes: 3 additions & 1 deletion src/global.H
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason extern int cudaGlobalDevice; now comes earlier, or just convenience?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, cudaGlobalDevice should be defined independently of whether nvcc is guiding the compile. It's not about earlier but getting it out of the __NVCC__ block.

Original file line number Diff line number Diff line change
Expand Up @@ -345,9 +345,11 @@ extern bool cuda_prof;
#include "Species.H"

#if HAVE_LIBCUDA==1
extern int cudaGlobalDevice;
#if defined (__NVCC__)
#include <cudaUtil.cuH>
extern thrust::device_vector<int> cuDstepL, cuDstepN;
extern int cudaGlobalDevice;
#endif
#endif

#endif
15 changes: 11 additions & 4 deletions src/user/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,19 @@ set (common_INCLUDE_DIRS
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/..)

set (common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
set (common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
exputil EXPlib yaml-cpp)

if(ENABLE_CUDA)
list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
# set_source_files_properties(UserBar.cc UserDisk.cc UserHalo.cc
# UserLogPot.cc UserMNdisk.cc UserMW.cc UserTest.cc UserTestCuda.cc
# PROPERTIES LANGUAGE CUDA)
endif()

set(user_SRC UserTest.cc)
Expand All @@ -33,7 +40,7 @@ foreach(mlib ${USER_MODULES})
add_library(${mlib} ${${mlib}_SRC})
set_target_properties(${mlib} PROPERTIES OUTPUT_NAME ${mlib})
target_include_directories(${mlib} PUBLIC ${common_INCLUDE_DIRS})
target_link_libraries(${mlib} PUBLIC ${common_LINKLIBS})
target_link_libraries(${mlib} PUBLIC ${common_LINKLIB})
install(TARGETS ${mlib} DESTINATION lib/user)
endforeach()

7 changes: 6 additions & 1 deletion utils/Analysis/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@ if(PNG_FOUND)
endif()

if(ENABLE_CUDA)
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(SLURM_FOUND)
Expand Down
7 changes: 6 additions & 1 deletion utils/ICs/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,12 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil
${HDF5_CXX_LIBRARIES})

if(ENABLE_CUDA)
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(ENABLE_XDR AND TIRPC_FOUND)
Expand Down
5 changes: 2 additions & 3 deletions utils/ICs/initial.cc
Original file line number Diff line number Diff line change
Expand Up @@ -360,15 +360,15 @@ main(int ac, char **av)
double RMIN, RCYLMIN, RCYLMAX, SCSPH, RSPHSL, DMFAC, RFACTOR, SHFAC;
double X0, Y0, Z0, U0, V0, W0;
int RNUM, PNUM, TNUM, VFLAG, DFLAG;
bool expcond, LOGR, CHEBY, SELECT, DUMPCOEF;
bool expcond, LOGR, CHEBY, DUMPCOEF;
int CMAPR, CMAPZ, NCHEB, TCHEB, CMTYPE, NDR, NDZ, NHR, NHT, NDP;
int LMAX, NMAXH, NMAXD, MMAX, NUMX, NUMY, NOUT, NMAXLIM, NODD, DF;
int DIVERGE, DIVERGE2, SEED, itmax;
double DIVERGE_RFAC, DIVERGE_RFAC2;
double PPower, R_DF, DR_DF;
double Hratio, scale_height, scale_length, scale_lenfkN;
double disk_mass, gas_mass, gscal_length, ToomreQ, Temp, Tmin;
bool const_height, images, multi, SVD, basis, zeropos, zerovel;
bool const_height, images, multi, basis, zeropos, zerovel;
bool report, ignore, evolved, diskmodel;
int nhalo, ndisk, ngas, ngparam;
std::string hbods, dbods, gbods, suffix, centerfile, halofile1, halofile2;
Expand Down Expand Up @@ -870,7 +870,6 @@ main(int ac, char **av)
EmpCylSL::CMAPZ = CMAPZ;
EmpCylSL::VFLAG = VFLAG;
EmpCylSL::logarithmic = LOGR;
EmpCylSL::PCAVAR = SELECT;

// Create expansion only if needed . . .
std::shared_ptr<EmpCylSL> expandd;
Expand Down
7 changes: 6 additions & 1 deletion utils/MSSA/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>
${HDF5_INCLUDE_DIRS})

if(ENABLE_CUDA)
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(SLURM_FOUND)
Expand Down
7 changes: 6 additions & 1 deletion utils/PhaseSpace/CMakeLists.txt
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just checking -- is this true for all cuda >12? Or just 12.4 and up?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From the release notes, it's true for all of CUDA 12. It could be that some of this was deprecated for a few point releases before finally not working at 12.4.

Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@ if(PNG_FOUND)
endif()

if(ENABLE_CUDA)
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(ENABLE_XDR AND TIRPC_FOUND)
Expand Down
7 changes: 6 additions & 1 deletion utils/SL/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil
${VTK_LIBRARIES})

if(ENABLE_CUDA)
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(ENABLE_XDR AND TIRPC_FOUND)
Expand Down
7 changes: 6 additions & 1 deletion utils/Test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,12 @@ set(common_INCLUDE
${CMAKE_CURRENT_SOURCE_DIR}/..)

if(ENABLE_CUDA)
list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt )
list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
list(APPEND common_LINKLIB CUDA::nvtx3)
else ()
list(APPEND common_LINKLIB CUDA::nvToolsExt)
endif ()
endif()

if(ENABLE_XDR AND TIRPC_FOUND)
Expand Down