From 045ea19e5d207f8a747e5bf90747893560b3af61 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Jul 2021 19:20:39 -0400 Subject: [PATCH 1/9] build low and high precision at the same time We can only provide one package containing both precisions. BREAKING CHANGES: Python: Python package will build both precisions, and DP_FLOAT_PREC is now runtime envrionmental variables C++: CMake will build both library, which will be called something like libdeepmd and libdeepmd_low LAMMPS: generate two directory USER-DEEPMD and USER-DEEPMD_low ipi: generate two execuate dp_ipi and dp_ipi_low --- deepmd/env.py | 20 +++++++--- doc/getting-started.md | 8 +++- doc/install.md | 4 +- setup.py | 10 ----- source/CMakeLists.txt | 16 +++----- source/api_cc/CMakeLists.txt | 14 ++++++- source/cmake/cmake_lammps.cmake.in | 6 ++- source/config/run_config.ini | 1 - source/install/build_cc.sh | 5 --- source/install/build_lammps.sh | 8 +++- source/ipi/CMakeLists.txt | 20 +++++++--- source/lib/CMakeLists.txt | 28 +++++++++++--- source/lib/src/cuda/CMakeLists.txt | 12 ++++-- source/lib/src/rocm/CMakeLists.txt | 14 +++++-- source/lmp/env.sh.in | 4 +- source/md/CMakeLists.txt | 32 ++++++++++------ source/op/CMakeLists.txt | 61 +++++++++++++++++++----------- 17 files changed, 168 insertions(+), 95 deletions(-) diff --git a/deepmd/env.py b/deepmd/env.py index 5f5c344031..ccdccb7679 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -232,19 +232,29 @@ def _get_package_constants( TF_VERSION = GLOBAL_CONFIG["tf_version"] TF_CXX11_ABI_FLAG = int(GLOBAL_CONFIG["tf_cxx11_abi_flag"]) -op_module = get_module("libop_abi") -op_grads_module = get_module("libop_grads") - -if GLOBAL_CONFIG["precision"] == "-DHIGH_PREC": +# FLOAT_PREC +dp_float_prec = os.environ.get("DP_FLOAT_PREC", "high").lower() +if dp_float_prec in ("high", ""): + # default is high + op_module = get_module("libop_abi") + op_grads_module = get_module("libop_grads") GLOBAL_TF_FLOAT_PRECISION = tf.float64 GLOBAL_NP_FLOAT_PRECISION = np.float64 GLOBAL_ENER_FLOAT_PRECISION = np.float64 global_float_prec = "double" -else: +elif dp_float_prec == "low": + op_module = get_module("libop_abi_low") + op_grads_module = get_module("libop_grads_low") GLOBAL_TF_FLOAT_PRECISION = tf.float32 GLOBAL_NP_FLOAT_PRECISION = np.float32 GLOBAL_ENER_FLOAT_PRECISION = np.float64 global_float_prec = "float" +else: + raise RuntimeError( + "Unsupported float precision option: %s. Supported: high," + "low. Please set precision with environmental variable " + "DP_FLOAT_PREC." % dp_float_prec + ) def global_cvt_2_tf_float(xx: tf.Tensor) -> tf.Tensor: diff --git a/doc/getting-started.md b/doc/getting-started.md index 76ed8acd5b..724e547d15 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -134,6 +134,12 @@ export TF_INTER_OP_PARALLELISM_THREADS=2 dp train input.json ``` +One can set other environmental variables: + +| Environment variables | Allowed value | Default value | Usage | +| --------------------- | ---------------------- | ------------- | -------------------------- | +| DP_FLOAT_PREC | `high`, `low` | `high` | Control high (double) or low (float) precision of training. | + ### Training analysis with Tensorboard If enbled in json/yaml input file DeePMD-kit will create log files which can be @@ -425,7 +431,7 @@ kspace_modify gewald 0.45 Please notice that the DeePMD does nothing to the direct space part of the electrostatic interaction, because this part is assumed to be fitted in the DeePMD model (the direct space cut-off is thus the cut-off of the DeePMD model). The splitting parameter `gewald` is modified by the `kspace_modify` command. ### Run path-integral MD with i-PI -The i-PI works in a client-server model. The i-PI provides the server for integrating the replica positions of atoms, while the DeePMD-kit provides a client named `dp_ipi` that computes the interactions (including energy, force and virial). The server and client communicates via the Unix domain socket or the Internet socket. Installation instructions of i-PI can be found [here](install.md#install-i-pi). The client can be started by +The i-PI works in a client-server model. The i-PI provides the server for integrating the replica positions of atoms, while the DeePMD-kit provides a client named `dp_ipi` (or `dp_ipi_low` for low precision) that computes the interactions (including energy, force and virial). The server and client communicates via the Unix domain socket or the Internet socket. Installation instructions of i-PI can be found [here](install.md#install-i-pi). The client can be started by ```bash i-pi input.xml & dp_ipi water.json diff --git a/doc/install.md b/doc/install.md index a45d8c7d9f..d6ae4fe6b0 100644 --- a/doc/install.md +++ b/doc/install.md @@ -136,7 +136,6 @@ One may set the following environment variables before executing `pip`: | Environment variables | Allowed value | Default value | Usage | | --------------------- | ---------------------- | ------------- | -------------------------- | | DP_VARIANT | `cpu`, `cuda`, `rocm` | `cpu` | Build CPU variant or GPU variant with CUDA or ROCM support. | -| DP_FLOAT_PREC | `high`, `low` | `high` | Build high (double) or low (float) precision. | | CUDA_TOOLKIT_ROOT_DIR | Path | Detected automatically | The path to the CUDA toolkit directory. | | ROCM_ROOT | Path | Detected automatically | The path to the ROCM toolkit directory. | @@ -201,7 +200,6 @@ One may add the following arguments to `cmake`: | ------------------------ | ------------------- | ------------- | ------------------------| | -DTENSORFLOW_ROOT=<value> | Path | - | The Path to TensorFlow's C++ interface. | | -DCMAKE_INSTALL_PREFIX=<value> | Path | - | The Path where DeePMD-kit will be installed. | -| -DFLOAT_PREC=<value> | `high` or `low` | `high` | Build high (double) or low (float) precision. | | -DUSE_CUDA_TOOLKIT=<value> | `TRUE` or `FALSE` | `FALSE` | If `TRUE`, Build GPU support with CUDA toolkit. | | -DCUDA_TOOLKIT_ROOT_DIR=<value> | Path | Detected automatically | The path to the CUDA toolkit directory. | | -DUSE_ROCM_TOOLKIT=<value> | `TRUE` or `FALSE` | `FALSE` | If `TRUE`, Build GPU support with ROCM toolkit. | @@ -228,7 +226,7 @@ DeePMD-kit provide module for running MD simulation with LAMMPS. Now make the De cd $deepmd_source_dir/source/build make lammps ``` -DeePMD-kit will generate a module called `USER-DEEPMD` in the `build` directory. Now download the LAMMPS code (`29Oct2020` or later), and uncompress it: +DeePMD-kit will generate a module called `USER-DEEPMD` (high precision) or `USER-DEEPMD_low` (low precision) in the `build` directory. Now download the LAMMPS code (`29Oct2020` or later), and uncompress it: ```bash cd /some/workspace wget https://github.com/lammps/lammps/archive/stable_29Oct2020.tar.gz diff --git a/setup.py b/setup.py index f0eb88ec39..e48f9a3f3d 100644 --- a/setup.py +++ b/setup.py @@ -52,16 +52,6 @@ else: raise RuntimeError("Unsupported DP_VARIANT option: %s" % dp_variant) -# FLOAT_PREC -dp_float_prec = os.environ.get("DP_FLOAT_PREC", "").lower() -if dp_float_prec in ["high", "low"]: - cmake_args.append("-DFLOAT_PREC:STRING=%s" % dp_float_prec) -elif dp_float_prec == "": - # default is high - cmake_args.append("-DFLOAT_PREC:STRING=high") -else: - raise RuntimeError("Unsupported float precision option: %s" % dp_float_prec) - # get tensorflow spec tf_spec = find_spec("tensorflow") if not tf_spec: diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 1c89c3cf25..81acb5bf8f 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -162,17 +162,11 @@ if ((NOT DEFINED CMAKE_BUILD_TYPE) OR CMAKE_BUILD_TYPE STREQUAL "") endif () # set op prec -if (DEFINED FLOAT_PREC) - string ( TOLOWER ${FLOAT_PREC} lower_float_prec ) - if (lower_float_prec STREQUAL "high") - set(PREC_DEF "-DHIGH_PREC") - else () - set(PREC_DEF "") - endif () -else () - set(PREC_DEF "-DHIGH_PREC") -endif() -add_definitions (${PREC_DEF}) +set(HIGH_PREC_DEF "HIGH_PREC") +# this defination doesn't work, but leaving it empty will cause error +set(LOW_PREC_DEF "LOW_PREC") +set(HIGH_PREC_VARIANT "") +set(LOW_PREC_VARIANT "_low") # find openmp find_package(OpenMP) diff --git a/source/api_cc/CMakeLists.txt b/source/api_cc/CMakeLists.txt index cfdfce9b0e..4389fdfc92 100644 --- a/source/api_cc/CMakeLists.txt +++ b/source/api_cc/CMakeLists.txt @@ -1,5 +1,4 @@ # libmd -set (libname ${LIB_DEEPMD_CC}) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/include/version.h.in @@ -18,6 +17,10 @@ endif() file(GLOB LIB_SRC src/*.cc src/*.cpp) file(GLOB INC_SRC include/*.h ${CMAKE_CURRENT_BINARY_DIR}/version.h) + +function(_add_libapicc_variant variant_name prec_def) +set (libname "${LIB_DEEPMD_CC}${variant_name}") + add_library(${libname} SHARED ${LIB_SRC}) if (USE_CUDA_TOOLKIT) @@ -28,6 +31,11 @@ if (USE_ROCM_TOOLKIT) target_link_libraries (${libname} ${ROCM_LIBRARIES}) endif() +set_target_properties( + ${libname} + PROPERTIES + COMPILE_DEFINITIONS ${prec_def} +) install(TARGETS ${libname} DESTINATION lib/) @@ -36,3 +44,7 @@ install( DESTINATION include/deepmd ) +endfunction() + +_add_libapicc_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_libapicc_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/cmake/cmake_lammps.cmake.in b/source/cmake/cmake_lammps.cmake.in index bee237a1c7..48e79addb2 100644 --- a/source/cmake/cmake_lammps.cmake.in +++ b/source/cmake/cmake_lammps.cmake.in @@ -1,4 +1,5 @@ -set (LMP_INSTALL_PREFIX "@CMAKE_BINARY_DIR@/USER-DEEPMD") +function(_add_lammps_variant variant_name prec_def) +set (LMP_INSTALL_PREFIX "@CMAKE_BINARY_DIR@/USER-DEEPMD${variant_name}") file(READ "@CMAKE_BINARY_DIR@/lmp/lammps_install_list.txt" files) string(REGEX REPLACE "\n" "" files "${files}") @@ -21,3 +22,6 @@ file ( TYPE FILE FILES "@CMAKE_BINARY_DIR@/lmp/pair_deepmd.h" ) +endfunction() +_add_lammps_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_lammps_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/config/run_config.ini b/source/config/run_config.ini index bb04319e47..87f54ef169 100644 --- a/source/config/run_config.ini +++ b/source/config/run_config.ini @@ -8,5 +8,4 @@ TF_INCLUDE_DIR = @TensorFlow_INCLUDE_DIRS@ TF_LIBS = @TensorFlow_LIBRARY@ TF_VERSION = @TENSORFLOW_VERSION@ TF_CXX11_ABI_FLAG = @OP_CXX_ABI@ -PRECISION = @PREC_DEF@ MODEL_VERSION=@MODEL_VERSION@ diff --git a/source/install/build_cc.sh b/source/install/build_cc.sh index 2a606c61f8..5a79fd47db 100755 --- a/source/install/build_cc.sh +++ b/source/install/build_cc.sh @@ -1,10 +1,5 @@ set -e -if [ -z "$FLOAT_PREC" ] -then - FLOAT_PREC=high -fi - if [ "$DP_VARIANT" == "cuda" ] then CUDA_ARGS="-DUSE_CUDA_TOOLKIT=TRUE" diff --git a/source/install/build_lammps.sh b/source/install/build_lammps.sh index a4fee95f6f..bc80859ab7 100755 --- a/source/install/build_lammps.sh +++ b/source/install/build_lammps.sh @@ -44,9 +44,13 @@ cp -r ${BUILD_TMP_DIR2}/USER-DEEPMD/* ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/ mkdir -p ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build cd ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build if [ ${FLOAT_PREC} == "high" ]; then - export PREC_DEF="-DHIGH_PREC" + PREC_DEF="-DHIGH_PREC" + DPLIB="-ldeepmd_op -ldeepmd -ldeepmd_cc" +else + PREC_DEF="-DLOW_PREC" + DPLIB="-ldeepmd_op_low -ldeepmd_low -ldeepmd_cc_low" fi -cmake -C ../cmake/presets/all_off.cmake -D PKG_USER-DEEPMD=ON -D PKG_KSPACE=ON -D CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -D CMAKE_CXX_FLAGS="${PREC_DEF} -I${INSTALL_PREFIX}/include -L${INSTALL_PREFIX}/lib -Wl,--no-as-needed -lrt -ldeepmd_op -ldeepmd -ldeepmd_cc -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=${INSTALL_PREFIX}/lib" ../cmake +cmake -C ../cmake/presets/all_off.cmake -D PKG_USER-DEEPMD=ON -D PKG_KSPACE=ON -D CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -D CMAKE_CXX_FLAGS="${PREC_DEF} -I${INSTALL_PREFIX}/include -L${INSTALL_PREFIX}/lib -Wl,--no-as-needed -lrt ${DPLIB} -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=${INSTALL_PREFIX}/lib" ../cmake make -j${NPROC} make install diff --git a/source/ipi/CMakeLists.txt b/source/ipi/CMakeLists.txt index c60122da78..e63ce6b529 100644 --- a/source/ipi/CMakeLists.txt +++ b/source/ipi/CMakeLists.txt @@ -4,24 +4,32 @@ list (APPEND MD_INCLUDE_PATH "include") include_directories (${MD_INCLUDE_PATH}) file(GLOB IN_SRC src/*.cc src/*.c) -add_library(${LIB_DEEPMD_IPI} SHARED ${IN_SRC}) + +function(_add_ipi_variant variant_name prec_def) +set (ipiname "dp_ipi${variant_name}") +set (libipiname "${LIB_DEEPMD_IPI}${variant_name}") +add_library(${libipiname} SHARED ${IN_SRC}) set(DRIVER_SOURCE_FILES driver.cc) -add_executable(dp_ipi ${DRIVER_SOURCE_FILES}) -target_link_libraries(dp_ipi ${LIB_DEEPMD_IPI} ${LIB_DEEPMD_OP} ${LIB_DEEPMD_CC} ${LIB_DEEPMD} ${TensorFlow_LIBRARY}) +add_executable(${ipiname} ${DRIVER_SOURCE_FILES}) +target_link_libraries(${ipiname} ${libipiname} ${LIB_DEEPMD_OP}${variant_name} ${LIB_DEEPMD_CC}${variant_name} ${LIB_DEEPMD}${variant_name} ${TensorFlow_LIBRARY}) set_target_properties( - dp_ipi + ${ipiname} PROPERTIES LINK_FLAGS "-Wl,-rpath,'$ORIGIN'/../lib -Wl,-z,defs" INSTALL_RPATH "$ORIGIN/../lib:${TensorFlow_LIBRARY_PATH}" + COMPILE_DEFINITIONS ${prec_def} ) install( - TARGETS ${LIB_DEEPMD_IPI} + TARGETS ${libipiname} DESTINATION lib/ ) install( - TARGETS dp_ipi + TARGETS ${ipiname} DESTINATION bin/ ) +endfunction() +_add_ipi_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_ipi_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") \ No newline at end of file diff --git a/source/lib/CMakeLists.txt b/source/lib/CMakeLists.txt index c1369c23ee..8e53f40862 100644 --- a/source/lib/CMakeLists.txt +++ b/source/lib/CMakeLists.txt @@ -12,22 +12,37 @@ endif() file(GLOB LIB_SRC src/*.cc src/*.cpp) file(GLOB INC_SRC include/*.h ${CMAKE_CURRENT_BINARY_DIR}/version.h) -add_library(${libname} SHARED ${LIB_SRC}) - if (USE_CUDA_TOOLKIT) add_definitions("-DGOOGLE_CUDA") add_subdirectory(src/cuda) - set (EXTRA_LIBS ${EXTRA_LIBS} deepmd_op_cuda) - target_link_libraries (${libname} ${CUDA_LIBRARIES} ${EXTRA_LIBS}) endif() if (USE_ROCM_TOOLKIT) add_definitions("-DTENSORFLOW_USE_ROCM") add_subdirectory(src/rocm) - set (EXTRA_LIBS ${EXTRA_LIBS} deepmd_op_rocm) +endif() + +function(_add_libdeepmd_variant variant_name prec_def) +set (libname "${LIB_DEEPMD}${variant_name}") + +add_library(${libname} SHARED ${LIB_SRC}) + +if (USE_CUDA_TOOLKIT) + set (EXTRA_LIBS ${EXTRA_LIBS} "deepmd_op_cuda${variant_name}") + target_link_libraries (${libname} ${CUDA_LIBRARIES} ${EXTRA_LIBS}) +endif() + +if (USE_ROCM_TOOLKIT) + set (EXTRA_LIBS ${EXTRA_LIBS} "deepmd_op_rocm${variant_name}") target_link_libraries (${libname} ${ROCM_LIBRARIES} ${EXTRA_LIBS}) endif() +set_target_properties( + ${libname} + PROPERTIES + COMPILE_DEFINITIONS ${prec_def} +) + if(BUILD_PY_IF) install(TARGETS ${libname} DESTINATION deepmd/op/) endif(BUILD_PY_IF) @@ -39,3 +54,6 @@ if(BUILD_CPP_IF) ) endif(BUILD_CPP_IF) +endfunction() +_add_libdeepmd_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_libdeepmd_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/lib/src/cuda/CMakeLists.txt b/source/lib/src/cuda/CMakeLists.txt index 2d9aa03cf6..1560602c5c 100644 --- a/source/lib/src/cuda/CMakeLists.txt +++ b/source/lib/src/cuda/CMakeLists.txt @@ -122,11 +122,17 @@ endif() file (GLOB SOURCE_FILES "*.cu" ) -cuda_add_library(deepmd_op_cuda SHARED ${SOURCE_FILES}) +function(_add_libcuda_variant variant_name prec_def) +set (libname "deepmd_op_cuda${variant_name}") +cuda_add_library(${libname} SHARED ${SOURCE_FILES} OPTIONS "-D${prec_def}") if (BUILD_CPP_IF) - install(TARGETS deepmd_op_cuda DESTINATION lib/) + install(TARGETS ${libname} DESTINATION lib/) endif (BUILD_CPP_IF) if (BUILD_PY_IF) - install(TARGETS deepmd_op_cuda DESTINATION deepmd/op/) + install(TARGETS ${libname} DESTINATION deepmd/op/) endif (BUILD_PY_IF) +endfunction() + +_add_libcuda_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_libcuda_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/lib/src/rocm/CMakeLists.txt b/source/lib/src/rocm/CMakeLists.txt index 393844b8bb..935ad99365 100644 --- a/source/lib/src/rocm/CMakeLists.txt +++ b/source/lib/src/rocm/CMakeLists.txt @@ -16,12 +16,18 @@ set (HIP_HIPCC_FLAGS -hc; -fno-gpu-rdc; --amdgpu-target=gfx906; -fPIC; -O3; --st file (GLOB SOURCE_FILES "*.hip.cu" ) -hip_add_library(deepmd_op_rocm SHARED ${SOURCE_FILES}) +function(_add_librocm_variant variant_name prec_def) +set (libname "deepmd_op_rocm${variant_name}") +hip_add_library(${libname} SHARED ${SOURCE_FILES} OPTIONS "-D${prec_def}") -install(TARGETS deepmd_op_rocm DESTINATION lib/) +install(TARGETS ${libname} DESTINATION lib/) if (BUILD_CPP_IF) - install(TARGETS deepmd_op_rocm DESTINATION lib/) + install(TARGETS ${libname} DESTINATION lib/) endif (BUILD_CPP_IF) if (BUILD_PY_IF) - install(TARGETS deepmd_op_rocm DESTINATION deepmd/op/) + install(TARGETS ${libname} DESTINATION deepmd/op/) endif (BUILD_PY_IF) +endfunction() + +_add_librocm_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_librocm_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/lmp/env.sh.in b/source/lmp/env.sh.in index 8157ddb8dd..1574b9620e 100644 --- a/source/lmp/env.sh.in +++ b/source/lmp/env.sh.in @@ -6,6 +6,6 @@ TF_INCLUDE_DIRS=`echo $TENSORFLOW_INCLUDE_DIRS | sed "s/;/ -I/g"` TF_LIBRARY_PATH=`echo $TENSORFLOW_LIBRARY_PATH | sed "s/;/ -L/g"` TF_RPATH=`echo $TENSORFLOW_LIBRARY_PATH | sed "s/;/ -Wl,-rpath=/g"` -NNP_INC=" -std=c++11 @PREC_DEF@ @TTM_DEF@ @OLD_LMP_PPPM_DEF@ -I$TF_INCLUDE_DIRS -I$DEEPMD_ROOT/include/ " +NNP_INC=" -std=c++11 -D@prec_def@ @TTM_DEF@ @OLD_LMP_PPPM_DEF@ -I$TF_INCLUDE_DIRS -I$DEEPMD_ROOT/include/ " NNP_PATH=" -L$TF_LIBRARY_PATH -L$DEEPMD_ROOT/lib" -NNP_LIB=" -Wl,--no-as-needed -l@LIB_DEEPMD_OP_DEVICE@ -l@LIB_DEEPMD_OP@ -l@LIB_DEEPMD_CC@ -l@LIB_DEEPMD@ -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=$TF_RPATH -Wl,-rpath=$DEEPMD_ROOT/lib" +NNP_LIB=" -Wl,--no-as-needed -l@LIB_DEEPMD_OP_DEVICE@@variant_name@ -l@LIB_DEEPMD_OP@@variant_name@ -l@LIB_DEEPMD_CC@@variant_name@ -l@LIB_DEEPMD@@variant_name@ -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=$TF_RPATH -Wl,-rpath=$DEEPMD_ROOT/lib" diff --git a/source/md/CMakeLists.txt b/source/md/CMakeLists.txt index 2db205dda2..4de0c00d6e 100644 --- a/source/md/CMakeLists.txt +++ b/source/md/CMakeLists.txt @@ -17,33 +17,38 @@ if (MAKE_FF_AD) set(MDFF_SOURCE_FILES mdff.cc) endif() -add_executable(dp_mdnn ${MDNN_SOURCE_FILES}) +function(_add_md_variant variant_name prec_def) +set (dp_mdnn_name "dp_mdnn${variant_name}") +set (dp_mdff_name "dp_mdff${variant_name}") +set (dp_mdad_name "dp_mdad${variant_name}") + +add_executable(${dp_mdnn_name} ${MDNN_SOURCE_FILES}) if (MAKE_FF_AD) - add_executable(dp_mdff ${MDFF_SOURCE_FILES}) - add_executable(dp_mdad ${MDAD_SOURCE_FILES}) + add_executable(${dp_mdff_name} ${MDFF_SOURCE_FILES}) + add_executable(${dp_mdad_name} ${MDAD_SOURCE_FILES}) endif() -target_link_libraries(dp_mdnn ${LIB_DEEPMD_NATIVE} ${LIB_DEEPMD_OP} ${LIB_DEEPMD} ${XDRFILE_LIBRARIES} ${TensorFlow_LIBRARY}) +target_link_libraries(${dp_mdnn_name} ${LIB_DEEPMD_NATIVE} ${LIB_DEEPMD_OP} ${LIB_DEEPMD} ${XDRFILE_LIBRARIES} ${TensorFlow_LIBRARY}) if (MAKE_FF_AD) - target_link_libraries(dp_mdad ${LIB_DEEPMD_NATIVE} ${LIB_DEEPMD_OP} ${LIB_DEEPMD} ${XDRFILE_LIBRARIES} ${TensorFlow_LIBRARY}) - target_link_libraries(dp_mdff ${LIB_DEEPMD_NATIVE} ${LIB_DEEPMD} ${XDRFILE_LIBRARIES} ${TensorFlow_LIBRARY}) + target_link_libraries(${dp_mdad_name} ${LIB_DEEPMD_NATIVE} ${LIB_DEEPMD_OP} ${LIB_DEEPMD} ${XDRFILE_LIBRARIES} ${TensorFlow_LIBRARY}) + target_link_libraries(${dp_mdff_name} ${LIB_DEEPMD_NATIVE} ${LIB_DEEPMD} ${XDRFILE_LIBRARIES} ${TensorFlow_LIBRARY}) endif() set_target_properties( - dp_mdnn + ${dp_mdnn_name} PROPERTIES LINK_FLAGS "-Wl,-rpath,'$ORIGIN'/../lib -Wl,-z,defs" INSTALL_RPATH "$ORIGIN/../lib:${TensorFlow_LIBRARY_PATH}" ) if (MAKE_FF_AD) set_target_properties( - dp_mdad + ${dp_mdad_name} PROPERTIES LINK_FLAGS "-Wl,-rpath,'$ORIGIN'/../lib -Wl,-z,defs" INSTALL_RPATH "$ORIGIN/../lib:${TensorFlow_LIBRARY_PATH}" ) set_target_properties( - dp_mdff + ${dp_mdff_name} PROPERTIES LINK_FLAGS "-Wl,-rpath,'$ORIGIN'/../lib -Wl,-z,defs" INSTALL_RPATH "$ORIGIN/../lib:${TensorFlow_LIBRARY_PATH}" @@ -55,16 +60,19 @@ install( DESTINATION lib/ ) install( - TARGETS dp_mdnn + TARGETS ${dp_mdnn_name} DESTINATION bin/ ) if (MAKE_FF_AD) install( - TARGETS dp_mdad + TARGETS ${dp_mdad_name} DESTINATION bin/ ) install( - TARGETS dp_mdff + TARGETS ${dp_mdff_name} DESTINATION bin/ ) endif() +endfunction() +_add_md_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_md_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/op/CMakeLists.txt b/source/op/CMakeLists.txt index 340c5601fb..7a34c257e5 100644 --- a/source/op/CMakeLists.txt +++ b/source/op/CMakeLists.txt @@ -9,62 +9,74 @@ file(GLOB OP_ROCM_SRC prod_force.cc prod_virial.cc descrpt.cc prod_env_mat_multi file(GLOB OP_GRADS_SRC prod_force_grad.cc prod_force_grad_multi_device.cc prod_virial_grad.cc prod_virial_grad_multi_device.cc soft_min_force_grad.cc soft_min_virial_grad.cc ) file(GLOB OP_PY *.py) +function(_add_libop_variant variant_name prec_def) +set(LIB_DEEPMD_OP_VARIANT "${LIB_DEEPMD_OP}${variant_name}") +set(LIB_OB_ABI_VARIANT "op_abi${variant_name}") +set(LIB_OB_GRADS_VARIANT "op_grads${variant_name}") + if (BUILD_CPP_IF) if (USE_CUDA_TOOLKIT) - add_library(${LIB_DEEPMD_OP} SHARED ${OP_CUDA_SRC}) + add_library(${LIB_DEEPMD_OP_VARIANT} SHARED ${OP_CUDA_SRC}) find_package(CUDA REQUIRED) include_directories(${CUDA_INCLUDE_DIRS}) - target_link_libraries (${LIB_DEEPMD_OP} ${CUDA_LIBRARIES}) + target_link_libraries (${LIB_DEEPMD_OP_VARIANT} ${CUDA_LIBRARIES}) elseif (USE_ROCM_TOOLKIT) - add_library(${LIB_DEEPMD_OP} SHARED ${OP_ROCM_SRC}) + add_library(${LIB_DEEPMD_OP_VARIANT} SHARED ${OP_ROCM_SRC}) find_package(ROCM REQUIRED) include_directories(${ROCM_INCLUDE_DIRS}) - target_link_libraries (${LIB_DEEPMD_OP} ${ROCM_LIBRARIES}) + target_link_libraries (${LIB_DEEPMD_OP_VARIANT} ${ROCM_LIBRARIES}) else () - add_library(${LIB_DEEPMD_OP} SHARED ${OP_SRC}) + add_library(${LIB_DEEPMD_OP_VARIANT} SHARED ${OP_SRC}) endif() + set_target_properties( + ${LIB_DEEPMD_OP_VARIANT} + PROPERTIES + COMPILE_DEFINITIONS ${prec_def} + ) endif (BUILD_CPP_IF) if (BUILD_PY_IF) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) set(CMAKE_INSTALL_RPATH $ORIGIN) if (USE_CUDA_TOOLKIT) - add_library(op_abi SHARED ${OP_SRC} ${OP_LIB}) - add_library(op_grads SHARED ${OP_GRADS_SRC}) + add_library(${LIB_OB_ABI_VARIANT} SHARED ${OP_SRC} ${OP_LIB}) + add_library(${LIB_OB_GRADS_VARIANT} SHARED ${OP_GRADS_SRC}) find_package(CUDA REQUIRED) include_directories(${CUDA_INCLUDE_DIRS}) - target_link_libraries (op_abi ${LIB_DEEPMD_OP_DEVICE}) - target_link_libraries (op_grads ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (${LIB_OB_ABI_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (${LIB_OB_GRADS_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) elseif(USE_ROCM_TOOLKIT) - add_library(op_abi SHARED ${OP_SRC} ${OP_LIB}) - add_library(op_grads SHARED ${OP_GRADS_SRC}) + add_library(${LIB_OB_ABI_VARIANT} SHARED ${OP_SRC} ${OP_LIB}) + add_library(${LIB_OB_GRADS_VARIANT} SHARED ${OP_GRADS_SRC}) find_package(ROCM REQUIRED) include_directories(${ROCM_INCLUDE_DIRS}) - target_link_libraries (op_abi ${LIB_DEEPMD_OP_DEVICE}) - target_link_libraries (op_grads ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (${LIB_OB_ABI_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (${LIB_OB_GRADS_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) else() - add_library(op_abi SHARED ${OP_SRC} ${OP_LIB}) - add_library(op_grads SHARED ${OP_GRADS_SRC}) + add_library(${LIB_OB_ABI_VARIANT} SHARED ${OP_SRC} ${OP_LIB}) + add_library(${LIB_OB_GRADS_VARIANT} SHARED ${OP_GRADS_SRC}) endif() message(STATUS ${TensorFlowFramework_LIBRARY}) - target_link_libraries(op_abi ${LIB_DEEPMD}) - target_link_libraries(op_grads ${LIB_DEEPMD}) + target_link_libraries(${LIB_OB_ABI_VARIANT} ${LIB_DEEPMD}) + target_link_libraries(${LIB_OB_GRADS_VARIANT} ${LIB_DEEPMD}) target_link_libraries( - op_abi ${TensorFlowFramework_LIBRARY} + ${LIB_OB_ABI_VARIANT} ${TensorFlowFramework_LIBRARY} ) target_link_libraries( - op_grads ${TensorFlowFramework_LIBRARY} + ${LIB_OB_GRADS_VARIANT} ${TensorFlowFramework_LIBRARY} ) set_target_properties( - op_abi + ${LIB_OB_ABI_VARIANT} PROPERTIES COMPILE_FLAGS ${OP_CXX_FLAG} + COMPILE_DEFINITIONS ${prec_def} ) set_target_properties( - op_grads + ${LIB_OB_GRADS_VARIANT} PROPERTIES COMPILE_FLAGS ${OP_CXX_FLAG} + COMPILE_DEFINITIONS ${prec_def} ) endif (BUILD_PY_IF) @@ -72,8 +84,11 @@ if (BUILD_CPP_IF) install(TARGETS ${LIB_DEEPMD_OP} DESTINATION lib/) endif (BUILD_CPP_IF) if (BUILD_PY_IF) - install(TARGETS op_abi DESTINATION deepmd/op/) - install(TARGETS op_grads DESTINATION deepmd/op/) + install(TARGETS ${LIB_OB_ABI_VARIANT} DESTINATION deepmd/op/) + install(TARGETS ${LIB_OB_GRADS_VARIANT} DESTINATION deepmd/op/) install(FILES ${OP_PY} DESTINATION deepmd/op/) endif (BUILD_PY_IF) +endfunction() +_add_libop_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_libop_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") From 73c9fdc768b24d4588e49147277a3dc63d555610 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Jul 2021 19:38:48 -0400 Subject: [PATCH 2/9] fix LAMMPS build script --- source/install/build_lammps.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/source/install/build_lammps.sh b/source/install/build_lammps.sh index bc80859ab7..15303d9d30 100755 --- a/source/install/build_lammps.sh +++ b/source/install/build_lammps.sh @@ -6,6 +6,14 @@ if [ -z "$FLOAT_PREC" ] then FLOAT_PREC=high fi + +if [ ${FLOAT_PREC} == "high" ]; then + PREC_DEF="-DHIGH_PREC" + PREC_SUFFIX="" +else + PREC_DEF="-DLOW_PREC" + PREC_SUFFIX="_low" +fi #------------------ SCRIPT_PATH=$(dirname $(realpath -s $0)) @@ -39,18 +47,11 @@ curl -L -o lammps.patch https://github.com/deepmd-kit-recipes/lammps-dp-feedstoc cd ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION} patch -f -p1 < ../lammps.patch || true mkdir -p ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/src/USER-DEEPMD -cp -r ${BUILD_TMP_DIR2}/USER-DEEPMD/* ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/src/USER-DEEPMD +cp -r ${BUILD_TMP_DIR2}/USER-DEEPMD${PREC_SUFFIX}/* ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/src/USER-DEEPMD mkdir -p ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build cd ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build -if [ ${FLOAT_PREC} == "high" ]; then - PREC_DEF="-DHIGH_PREC" - DPLIB="-ldeepmd_op -ldeepmd -ldeepmd_cc" -else - PREC_DEF="-DLOW_PREC" - DPLIB="-ldeepmd_op_low -ldeepmd_low -ldeepmd_cc_low" -fi -cmake -C ../cmake/presets/all_off.cmake -D PKG_USER-DEEPMD=ON -D PKG_KSPACE=ON -D CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -D CMAKE_CXX_FLAGS="${PREC_DEF} -I${INSTALL_PREFIX}/include -L${INSTALL_PREFIX}/lib -Wl,--no-as-needed -lrt ${DPLIB} -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=${INSTALL_PREFIX}/lib" ../cmake +cmake -C ../cmake/presets/all_off.cmake -D PKG_USER-DEEPMD=ON -D PKG_KSPACE=ON -D CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -D CMAKE_CXX_FLAGS="${PREC_DEF} -I${INSTALL_PREFIX}/include -L${INSTALL_PREFIX}/lib -Wl,--no-as-needed -lrt -ldeepmd_op${PREC_SUFFIX} -ldeepmd${PREC_SUFFIX} -ldeepmd_cc${PREC_SUFFIX} -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=${INSTALL_PREFIX}/lib" ../cmake make -j${NPROC} make install From 0a3c97b6ffeb21f7d405bf8c8e281ccd69ae0e1d Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Jul 2021 19:59:12 -0400 Subject: [PATCH 3/9] fix lammps cmake file --- source/cmake/cmake_lammps.cmake.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/cmake/cmake_lammps.cmake.in b/source/cmake/cmake_lammps.cmake.in index 48e79addb2..fe011eaf3a 100644 --- a/source/cmake/cmake_lammps.cmake.in +++ b/source/cmake/cmake_lammps.cmake.in @@ -23,5 +23,5 @@ file ( FILES "@CMAKE_BINARY_DIR@/lmp/pair_deepmd.h" ) endfunction() -_add_lammps_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") -_add_lammps_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") +_add_lammps_variant("@HIGH_PREC_VARIANT@" "@HIGH_PREC_DEF@") +_add_lammps_variant("@LOW_PREC_VARIANT@" "@LOW_PREC_DEF@") From bbc48b047a91006312eaeab04eabb01a7cbedaf7 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Jul 2021 20:16:37 -0400 Subject: [PATCH 4/9] install LIB_DEEPMD_OP_VARIANT --- source/op/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/op/CMakeLists.txt b/source/op/CMakeLists.txt index 7a34c257e5..6059cfc99d 100644 --- a/source/op/CMakeLists.txt +++ b/source/op/CMakeLists.txt @@ -81,7 +81,7 @@ if (BUILD_PY_IF) endif (BUILD_PY_IF) if (BUILD_CPP_IF) - install(TARGETS ${LIB_DEEPMD_OP} DESTINATION lib/) + install(TARGETS ${LIB_DEEPMD_OP_VARIANT} DESTINATION lib/) endif (BUILD_CPP_IF) if (BUILD_PY_IF) install(TARGETS ${LIB_OB_ABI_VARIANT} DESTINATION deepmd/op/) From 715a7cf5926da38db752b3dad95362c2d13e0640 Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Jul 2021 21:10:06 -0400 Subject: [PATCH 5/9] remove FLOAT_PREC argument --- source/install/build_cc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/install/build_cc.sh b/source/install/build_cc.sh index 5a79fd47db..dafcde7606 100755 --- a/source/install/build_cc.sh +++ b/source/install/build_cc.sh @@ -20,7 +20,7 @@ NPROC=$(nproc --all) BUILD_TMP_DIR=${SCRIPT_PATH}/../build mkdir -p ${BUILD_TMP_DIR} cd ${BUILD_TMP_DIR} -cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DFLOAT_PREC=${FLOAT_PREC} -DINSTALL_TENSORFLOW=TRUE ${CUDA_ARGS} .. +cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DINSTALL_TENSORFLOW=TRUE ${CUDA_ARGS} .. make -j${NPROC} make install From d896605e841077dbeab131615640d87e65ccd9dc Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Jul 2021 22:39:07 -0400 Subject: [PATCH 6/9] change DP_FLOAT_PREC to DP_INTERFACE_PREC --- deepmd/env.py | 2 +- doc/getting-started.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepmd/env.py b/deepmd/env.py index ccdccb7679..d558627a2a 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -233,7 +233,7 @@ def _get_package_constants( TF_CXX11_ABI_FLAG = int(GLOBAL_CONFIG["tf_cxx11_abi_flag"]) # FLOAT_PREC -dp_float_prec = os.environ.get("DP_FLOAT_PREC", "high").lower() +dp_float_prec = os.environ.get("DP_INTERFACE_PREC", "high").lower() if dp_float_prec in ("high", ""): # default is high op_module = get_module("libop_abi") diff --git a/doc/getting-started.md b/doc/getting-started.md index 724e547d15..4a4a1202e4 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -138,7 +138,7 @@ One can set other environmental variables: | Environment variables | Allowed value | Default value | Usage | | --------------------- | ---------------------- | ------------- | -------------------------- | -| DP_FLOAT_PREC | `high`, `low` | `high` | Control high (double) or low (float) precision of training. | +| DP_INTERFACE_PREC | `high`, `low` | `high` | Control high (double) or low (float) precision of training. | ### Training analysis with Tensorboard From 153a754365af39675e6f629db3aeb2778958ad9f Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 22 Jul 2021 23:06:45 -0400 Subject: [PATCH 7/9] revert some libraries as they do not need to build twice --- deepmd/env.py | 7 ++-- source/install/build_lammps.sh | 2 +- source/ipi/CMakeLists.txt | 2 +- source/lib/CMakeLists.txt | 28 +++---------- source/lib/src/cuda/CMakeLists.txt | 12 ++---- source/lib/src/rocm/CMakeLists.txt | 14 ++----- source/lmp/env.sh.in | 2 +- source/op/CMakeLists.txt | 63 ++++++++++++------------------ 8 files changed, 42 insertions(+), 88 deletions(-) diff --git a/deepmd/env.py b/deepmd/env.py index d558627a2a..045467fe7b 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -232,19 +232,18 @@ def _get_package_constants( TF_VERSION = GLOBAL_CONFIG["tf_version"] TF_CXX11_ABI_FLAG = int(GLOBAL_CONFIG["tf_cxx11_abi_flag"]) +op_module = get_module("libop_abi") +op_grads_module = get_module("libop_grads") + # FLOAT_PREC dp_float_prec = os.environ.get("DP_INTERFACE_PREC", "high").lower() if dp_float_prec in ("high", ""): # default is high - op_module = get_module("libop_abi") - op_grads_module = get_module("libop_grads") GLOBAL_TF_FLOAT_PRECISION = tf.float64 GLOBAL_NP_FLOAT_PRECISION = np.float64 GLOBAL_ENER_FLOAT_PRECISION = np.float64 global_float_prec = "double" elif dp_float_prec == "low": - op_module = get_module("libop_abi_low") - op_grads_module = get_module("libop_grads_low") GLOBAL_TF_FLOAT_PRECISION = tf.float32 GLOBAL_NP_FLOAT_PRECISION = np.float32 GLOBAL_ENER_FLOAT_PRECISION = np.float64 diff --git a/source/install/build_lammps.sh b/source/install/build_lammps.sh index 15303d9d30..724b7bf6e9 100755 --- a/source/install/build_lammps.sh +++ b/source/install/build_lammps.sh @@ -51,7 +51,7 @@ cp -r ${BUILD_TMP_DIR2}/USER-DEEPMD${PREC_SUFFIX}/* ${BUILD_TMP_DIR}/lammps-${LA mkdir -p ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build cd ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build -cmake -C ../cmake/presets/all_off.cmake -D PKG_USER-DEEPMD=ON -D PKG_KSPACE=ON -D CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -D CMAKE_CXX_FLAGS="${PREC_DEF} -I${INSTALL_PREFIX}/include -L${INSTALL_PREFIX}/lib -Wl,--no-as-needed -lrt -ldeepmd_op${PREC_SUFFIX} -ldeepmd${PREC_SUFFIX} -ldeepmd_cc${PREC_SUFFIX} -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=${INSTALL_PREFIX}/lib" ../cmake +cmake -C ../cmake/presets/all_off.cmake -D PKG_USER-DEEPMD=ON -D PKG_KSPACE=ON -D CMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -D CMAKE_CXX_FLAGS="${PREC_DEF} -I${INSTALL_PREFIX}/include -L${INSTALL_PREFIX}/lib -Wl,--no-as-needed -lrt -ldeepmd_op -ldeepmd -ldeepmd_cc${PREC_SUFFIX} -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=${INSTALL_PREFIX}/lib" ../cmake make -j${NPROC} make install diff --git a/source/ipi/CMakeLists.txt b/source/ipi/CMakeLists.txt index e63ce6b529..cdb3f80b08 100644 --- a/source/ipi/CMakeLists.txt +++ b/source/ipi/CMakeLists.txt @@ -12,7 +12,7 @@ add_library(${libipiname} SHARED ${IN_SRC}) set(DRIVER_SOURCE_FILES driver.cc) add_executable(${ipiname} ${DRIVER_SOURCE_FILES}) -target_link_libraries(${ipiname} ${libipiname} ${LIB_DEEPMD_OP}${variant_name} ${LIB_DEEPMD_CC}${variant_name} ${LIB_DEEPMD}${variant_name} ${TensorFlow_LIBRARY}) +target_link_libraries(${ipiname} ${libipiname} ${LIB_DEEPMD_OP} ${LIB_DEEPMD_CC}${variant_name} ${LIB_DEEPMD} ${TensorFlow_LIBRARY}) set_target_properties( ${ipiname} diff --git a/source/lib/CMakeLists.txt b/source/lib/CMakeLists.txt index 8e53f40862..c1369c23ee 100644 --- a/source/lib/CMakeLists.txt +++ b/source/lib/CMakeLists.txt @@ -12,37 +12,22 @@ endif() file(GLOB LIB_SRC src/*.cc src/*.cpp) file(GLOB INC_SRC include/*.h ${CMAKE_CURRENT_BINARY_DIR}/version.h) +add_library(${libname} SHARED ${LIB_SRC}) + if (USE_CUDA_TOOLKIT) add_definitions("-DGOOGLE_CUDA") add_subdirectory(src/cuda) + set (EXTRA_LIBS ${EXTRA_LIBS} deepmd_op_cuda) + target_link_libraries (${libname} ${CUDA_LIBRARIES} ${EXTRA_LIBS}) endif() if (USE_ROCM_TOOLKIT) add_definitions("-DTENSORFLOW_USE_ROCM") add_subdirectory(src/rocm) -endif() - -function(_add_libdeepmd_variant variant_name prec_def) -set (libname "${LIB_DEEPMD}${variant_name}") - -add_library(${libname} SHARED ${LIB_SRC}) - -if (USE_CUDA_TOOLKIT) - set (EXTRA_LIBS ${EXTRA_LIBS} "deepmd_op_cuda${variant_name}") - target_link_libraries (${libname} ${CUDA_LIBRARIES} ${EXTRA_LIBS}) -endif() - -if (USE_ROCM_TOOLKIT) - set (EXTRA_LIBS ${EXTRA_LIBS} "deepmd_op_rocm${variant_name}") + set (EXTRA_LIBS ${EXTRA_LIBS} deepmd_op_rocm) target_link_libraries (${libname} ${ROCM_LIBRARIES} ${EXTRA_LIBS}) endif() -set_target_properties( - ${libname} - PROPERTIES - COMPILE_DEFINITIONS ${prec_def} -) - if(BUILD_PY_IF) install(TARGETS ${libname} DESTINATION deepmd/op/) endif(BUILD_PY_IF) @@ -54,6 +39,3 @@ if(BUILD_CPP_IF) ) endif(BUILD_CPP_IF) -endfunction() -_add_libdeepmd_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") -_add_libdeepmd_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/lib/src/cuda/CMakeLists.txt b/source/lib/src/cuda/CMakeLists.txt index 1560602c5c..2d9aa03cf6 100644 --- a/source/lib/src/cuda/CMakeLists.txt +++ b/source/lib/src/cuda/CMakeLists.txt @@ -122,17 +122,11 @@ endif() file (GLOB SOURCE_FILES "*.cu" ) -function(_add_libcuda_variant variant_name prec_def) -set (libname "deepmd_op_cuda${variant_name}") -cuda_add_library(${libname} SHARED ${SOURCE_FILES} OPTIONS "-D${prec_def}") +cuda_add_library(deepmd_op_cuda SHARED ${SOURCE_FILES}) if (BUILD_CPP_IF) - install(TARGETS ${libname} DESTINATION lib/) + install(TARGETS deepmd_op_cuda DESTINATION lib/) endif (BUILD_CPP_IF) if (BUILD_PY_IF) - install(TARGETS ${libname} DESTINATION deepmd/op/) + install(TARGETS deepmd_op_cuda DESTINATION deepmd/op/) endif (BUILD_PY_IF) -endfunction() - -_add_libcuda_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") -_add_libcuda_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/lib/src/rocm/CMakeLists.txt b/source/lib/src/rocm/CMakeLists.txt index 935ad99365..393844b8bb 100644 --- a/source/lib/src/rocm/CMakeLists.txt +++ b/source/lib/src/rocm/CMakeLists.txt @@ -16,18 +16,12 @@ set (HIP_HIPCC_FLAGS -hc; -fno-gpu-rdc; --amdgpu-target=gfx906; -fPIC; -O3; --st file (GLOB SOURCE_FILES "*.hip.cu" ) -function(_add_librocm_variant variant_name prec_def) -set (libname "deepmd_op_rocm${variant_name}") -hip_add_library(${libname} SHARED ${SOURCE_FILES} OPTIONS "-D${prec_def}") +hip_add_library(deepmd_op_rocm SHARED ${SOURCE_FILES}) -install(TARGETS ${libname} DESTINATION lib/) +install(TARGETS deepmd_op_rocm DESTINATION lib/) if (BUILD_CPP_IF) - install(TARGETS ${libname} DESTINATION lib/) + install(TARGETS deepmd_op_rocm DESTINATION lib/) endif (BUILD_CPP_IF) if (BUILD_PY_IF) - install(TARGETS ${libname} DESTINATION deepmd/op/) + install(TARGETS deepmd_op_rocm DESTINATION deepmd/op/) endif (BUILD_PY_IF) -endfunction() - -_add_librocm_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") -_add_librocm_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") diff --git a/source/lmp/env.sh.in b/source/lmp/env.sh.in index 1574b9620e..25ca8dc90c 100644 --- a/source/lmp/env.sh.in +++ b/source/lmp/env.sh.in @@ -8,4 +8,4 @@ TF_RPATH=`echo $TENSORFLOW_LIBRARY_PATH | sed "s/;/ -Wl,-rpath=/g"` NNP_INC=" -std=c++11 -D@prec_def@ @TTM_DEF@ @OLD_LMP_PPPM_DEF@ -I$TF_INCLUDE_DIRS -I$DEEPMD_ROOT/include/ " NNP_PATH=" -L$TF_LIBRARY_PATH -L$DEEPMD_ROOT/lib" -NNP_LIB=" -Wl,--no-as-needed -l@LIB_DEEPMD_OP_DEVICE@@variant_name@ -l@LIB_DEEPMD_OP@@variant_name@ -l@LIB_DEEPMD_CC@@variant_name@ -l@LIB_DEEPMD@@variant_name@ -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=$TF_RPATH -Wl,-rpath=$DEEPMD_ROOT/lib" +NNP_LIB=" -Wl,--no-as-needed -l@LIB_DEEPMD_OP_DEVICE@ -l@LIB_DEEPMD_OP@ -l@LIB_DEEPMD_CC@@variant_name@ -l@LIB_DEEPMD@ -ltensorflow_cc -ltensorflow_framework -Wl,-rpath=$TF_RPATH -Wl,-rpath=$DEEPMD_ROOT/lib" diff --git a/source/op/CMakeLists.txt b/source/op/CMakeLists.txt index 6059cfc99d..340c5601fb 100644 --- a/source/op/CMakeLists.txt +++ b/source/op/CMakeLists.txt @@ -9,86 +9,71 @@ file(GLOB OP_ROCM_SRC prod_force.cc prod_virial.cc descrpt.cc prod_env_mat_multi file(GLOB OP_GRADS_SRC prod_force_grad.cc prod_force_grad_multi_device.cc prod_virial_grad.cc prod_virial_grad_multi_device.cc soft_min_force_grad.cc soft_min_virial_grad.cc ) file(GLOB OP_PY *.py) -function(_add_libop_variant variant_name prec_def) -set(LIB_DEEPMD_OP_VARIANT "${LIB_DEEPMD_OP}${variant_name}") -set(LIB_OB_ABI_VARIANT "op_abi${variant_name}") -set(LIB_OB_GRADS_VARIANT "op_grads${variant_name}") - if (BUILD_CPP_IF) if (USE_CUDA_TOOLKIT) - add_library(${LIB_DEEPMD_OP_VARIANT} SHARED ${OP_CUDA_SRC}) + add_library(${LIB_DEEPMD_OP} SHARED ${OP_CUDA_SRC}) find_package(CUDA REQUIRED) include_directories(${CUDA_INCLUDE_DIRS}) - target_link_libraries (${LIB_DEEPMD_OP_VARIANT} ${CUDA_LIBRARIES}) + target_link_libraries (${LIB_DEEPMD_OP} ${CUDA_LIBRARIES}) elseif (USE_ROCM_TOOLKIT) - add_library(${LIB_DEEPMD_OP_VARIANT} SHARED ${OP_ROCM_SRC}) + add_library(${LIB_DEEPMD_OP} SHARED ${OP_ROCM_SRC}) find_package(ROCM REQUIRED) include_directories(${ROCM_INCLUDE_DIRS}) - target_link_libraries (${LIB_DEEPMD_OP_VARIANT} ${ROCM_LIBRARIES}) + target_link_libraries (${LIB_DEEPMD_OP} ${ROCM_LIBRARIES}) else () - add_library(${LIB_DEEPMD_OP_VARIANT} SHARED ${OP_SRC}) + add_library(${LIB_DEEPMD_OP} SHARED ${OP_SRC}) endif() - set_target_properties( - ${LIB_DEEPMD_OP_VARIANT} - PROPERTIES - COMPILE_DEFINITIONS ${prec_def} - ) endif (BUILD_CPP_IF) if (BUILD_PY_IF) set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) set(CMAKE_INSTALL_RPATH $ORIGIN) if (USE_CUDA_TOOLKIT) - add_library(${LIB_OB_ABI_VARIANT} SHARED ${OP_SRC} ${OP_LIB}) - add_library(${LIB_OB_GRADS_VARIANT} SHARED ${OP_GRADS_SRC}) + add_library(op_abi SHARED ${OP_SRC} ${OP_LIB}) + add_library(op_grads SHARED ${OP_GRADS_SRC}) find_package(CUDA REQUIRED) include_directories(${CUDA_INCLUDE_DIRS}) - target_link_libraries (${LIB_OB_ABI_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) - target_link_libraries (${LIB_OB_GRADS_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (op_abi ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (op_grads ${LIB_DEEPMD_OP_DEVICE}) elseif(USE_ROCM_TOOLKIT) - add_library(${LIB_OB_ABI_VARIANT} SHARED ${OP_SRC} ${OP_LIB}) - add_library(${LIB_OB_GRADS_VARIANT} SHARED ${OP_GRADS_SRC}) + add_library(op_abi SHARED ${OP_SRC} ${OP_LIB}) + add_library(op_grads SHARED ${OP_GRADS_SRC}) find_package(ROCM REQUIRED) include_directories(${ROCM_INCLUDE_DIRS}) - target_link_libraries (${LIB_OB_ABI_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) - target_link_libraries (${LIB_OB_GRADS_VARIANT} ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (op_abi ${LIB_DEEPMD_OP_DEVICE}) + target_link_libraries (op_grads ${LIB_DEEPMD_OP_DEVICE}) else() - add_library(${LIB_OB_ABI_VARIANT} SHARED ${OP_SRC} ${OP_LIB}) - add_library(${LIB_OB_GRADS_VARIANT} SHARED ${OP_GRADS_SRC}) + add_library(op_abi SHARED ${OP_SRC} ${OP_LIB}) + add_library(op_grads SHARED ${OP_GRADS_SRC}) endif() message(STATUS ${TensorFlowFramework_LIBRARY}) - target_link_libraries(${LIB_OB_ABI_VARIANT} ${LIB_DEEPMD}) - target_link_libraries(${LIB_OB_GRADS_VARIANT} ${LIB_DEEPMD}) + target_link_libraries(op_abi ${LIB_DEEPMD}) + target_link_libraries(op_grads ${LIB_DEEPMD}) target_link_libraries( - ${LIB_OB_ABI_VARIANT} ${TensorFlowFramework_LIBRARY} + op_abi ${TensorFlowFramework_LIBRARY} ) target_link_libraries( - ${LIB_OB_GRADS_VARIANT} ${TensorFlowFramework_LIBRARY} + op_grads ${TensorFlowFramework_LIBRARY} ) set_target_properties( - ${LIB_OB_ABI_VARIANT} + op_abi PROPERTIES COMPILE_FLAGS ${OP_CXX_FLAG} - COMPILE_DEFINITIONS ${prec_def} ) set_target_properties( - ${LIB_OB_GRADS_VARIANT} + op_grads PROPERTIES COMPILE_FLAGS ${OP_CXX_FLAG} - COMPILE_DEFINITIONS ${prec_def} ) endif (BUILD_PY_IF) if (BUILD_CPP_IF) - install(TARGETS ${LIB_DEEPMD_OP_VARIANT} DESTINATION lib/) + install(TARGETS ${LIB_DEEPMD_OP} DESTINATION lib/) endif (BUILD_CPP_IF) if (BUILD_PY_IF) - install(TARGETS ${LIB_OB_ABI_VARIANT} DESTINATION deepmd/op/) - install(TARGETS ${LIB_OB_GRADS_VARIANT} DESTINATION deepmd/op/) + install(TARGETS op_abi DESTINATION deepmd/op/) + install(TARGETS op_grads DESTINATION deepmd/op/) install(FILES ${OP_PY} DESTINATION deepmd/op/) endif (BUILD_PY_IF) -endfunction() -_add_libop_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") -_add_libop_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") From 59f15ca7d70d4f848c742899a663320ad76d94df Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Fri, 23 Jul 2021 03:24:42 -0400 Subject: [PATCH 8/9] update error message --- deepmd/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepmd/env.py b/deepmd/env.py index 045467fe7b..4e03aa4f0b 100644 --- a/deepmd/env.py +++ b/deepmd/env.py @@ -252,7 +252,7 @@ def _get_package_constants( raise RuntimeError( "Unsupported float precision option: %s. Supported: high," "low. Please set precision with environmental variable " - "DP_FLOAT_PREC." % dp_float_prec + "DP_INTERFACE_PREC." % dp_float_prec ) From b8c166bba1b396ea5cb6e83b4f5193ed298f442d Mon Sep 17 00:00:00 2001 From: Jinzhe Zeng Date: Thu, 29 Jul 2021 00:31:22 -0400 Subject: [PATCH 9/9] change the implementation of LAMMPS variant now `env.sh` and `env_low.sh` will be generated in the same directory. Users can easily `mv env_low.sh env.sh` if they need low precision. --- doc/install.md | 2 +- source/cmake/cmake_lammps.cmake.in | 13 +++++++------ source/install/build_lammps.sh | 2 +- source/lmp/CMakeLists.txt | 7 ++++++- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/doc/install.md b/doc/install.md index d6ae4fe6b0..d18bf8effa 100644 --- a/doc/install.md +++ b/doc/install.md @@ -226,7 +226,7 @@ DeePMD-kit provide module for running MD simulation with LAMMPS. Now make the De cd $deepmd_source_dir/source/build make lammps ``` -DeePMD-kit will generate a module called `USER-DEEPMD` (high precision) or `USER-DEEPMD_low` (low precision) in the `build` directory. Now download the LAMMPS code (`29Oct2020` or later), and uncompress it: +DeePMD-kit will generate a module called `USER-DEEPMD` in the `build` directory. If you need low precision version, move `env_low.sh` to `env.sh` in the directory. Now download the LAMMPS code (`29Oct2020` or later), and uncompress it: ```bash cd /some/workspace wget https://github.com/lammps/lammps/archive/stable_29Oct2020.tar.gz diff --git a/source/cmake/cmake_lammps.cmake.in b/source/cmake/cmake_lammps.cmake.in index fe011eaf3a..44ad206d9c 100644 --- a/source/cmake/cmake_lammps.cmake.in +++ b/source/cmake/cmake_lammps.cmake.in @@ -1,5 +1,4 @@ -function(_add_lammps_variant variant_name prec_def) -set (LMP_INSTALL_PREFIX "@CMAKE_BINARY_DIR@/USER-DEEPMD${variant_name}") +set (LMP_INSTALL_PREFIX "@CMAKE_BINARY_DIR@/USER-DEEPMD") file(READ "@CMAKE_BINARY_DIR@/lmp/lammps_install_list.txt" files) string(REGEX REPLACE "\n" "" files "${files}") @@ -14,7 +13,12 @@ endforeach () file ( INSTALL DESTINATION "${LMP_INSTALL_PREFIX}" TYPE FILE - FILES "@CMAKE_BINARY_DIR@/lmp/env.sh" + FILES "@CMAKE_BINARY_DIR@/lmp/env@HIGH_PREC_VARIANT@.sh" +) +file ( + INSTALL DESTINATION "${LMP_INSTALL_PREFIX}" + TYPE FILE + FILES "@CMAKE_BINARY_DIR@/lmp/env@LOW_PREC_VARIANT@.sh" ) file ( @@ -22,6 +26,3 @@ file ( TYPE FILE FILES "@CMAKE_BINARY_DIR@/lmp/pair_deepmd.h" ) -endfunction() -_add_lammps_variant("@HIGH_PREC_VARIANT@" "@HIGH_PREC_DEF@") -_add_lammps_variant("@LOW_PREC_VARIANT@" "@LOW_PREC_DEF@") diff --git a/source/install/build_lammps.sh b/source/install/build_lammps.sh index 724b7bf6e9..6bcec33685 100755 --- a/source/install/build_lammps.sh +++ b/source/install/build_lammps.sh @@ -47,7 +47,7 @@ curl -L -o lammps.patch https://github.com/deepmd-kit-recipes/lammps-dp-feedstoc cd ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION} patch -f -p1 < ../lammps.patch || true mkdir -p ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/src/USER-DEEPMD -cp -r ${BUILD_TMP_DIR2}/USER-DEEPMD${PREC_SUFFIX}/* ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/src/USER-DEEPMD +cp -r ${BUILD_TMP_DIR2}/USER-DEEPMD/* ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/src/USER-DEEPMD mkdir -p ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build cd ${BUILD_TMP_DIR}/lammps-${LAMMPS_VERSION}/build diff --git a/source/lmp/CMakeLists.txt b/source/lmp/CMakeLists.txt index bc52c8fa81..2c68352385 100644 --- a/source/lmp/CMakeLists.txt +++ b/source/lmp/CMakeLists.txt @@ -7,6 +7,11 @@ list (APPEND LMP_INSTALL_FILES ${LMP_HEADER}) list (APPEND LMP_INSTALL_FILES ${LMP_SRC}) list (APPEND LMP_INSTALL_FILES ${LMP_SHSCRIPT}) -configure_file("env.sh.in" "env.sh" @ONLY) +function(_add_lmp_variant variant_name prec_def) +configure_file("env.sh.in" "env${variant_name}.sh" @ONLY) +endfunction() +_add_lmp_variant("${HIGH_PREC_VARIANT}" "${HIGH_PREC_DEF}") +_add_lmp_variant("${LOW_PREC_VARIANT}" "${LOW_PREC_DEF}") + configure_file("pair_deepmd.h.in" "pair_deepmd.h" @ONLY) configure_file("lammps_install_list.txt.in" "lammps_install_list.txt" @ONLY)