diff --git a/CMakeLists.txt b/CMakeLists.txt index e83556525c..8ac95e45fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -807,4 +807,4 @@ install(PROGRAMS ${ABACUS_BIN_PATH} if(ENABLE_COVERAGE) coverage_evaluate() -endif() \ No newline at end of file +endif() diff --git a/Dockerfile.intel b/Dockerfile.intel index 5bf6308e79..119cf4eb4b 100644 --- a/Dockerfile.intel +++ b/Dockerfile.intel @@ -2,7 +2,8 @@ FROM ubuntu:22.04 RUN apt-get update && apt-get install -y \ bc cmake git gnupg gcc g++ python3-numpy sudo wget vim unzip \ - libcereal-dev libxc-dev libgtest-dev libgmock-dev libbenchmark-dev + libcereal-dev libxc-dev libgtest-dev libgmock-dev libbenchmark-dev \ + pkg-config build-essential autoconf automake libtool # Following steps by https://software.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/apt.html . RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \ @@ -16,21 +17,44 @@ RUN apt-get update && \ intel-oneapi-compiler-dpcpp-cpp \ intel-oneapi-compiler-fortran \ intel-oneapi-mkl-devel \ - intel-oneapi-mpi-devel="2021.11.*" \ + intel-oneapi-mkl-sycl-devel \ + intel-oneapi-mkl-sycl-distributed-dft-devel \ + intel-oneapi-mpi-devel \ intel-oneapi-vtune && \ - rm /opt/intel/oneapi/mpi/latest && ln -s /opt/intel/oneapi/mpi/2021.11 /opt/intel/oneapi/mpi/latest -ENV I_MPI_ROOT=/opt/intel/oneapi/mpi/latest \ - LIBRARY_PATH=/opt/intel/oneapi/tbb/latest/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib/:/opt/intel/oneapi/ippcp/latest/lib/:/opt/intel/oneapi/ipp/latest/lib:/opt/intel/oneapi/dpl/latest/lib:/opt/intel/oneapi/dnnl/latest/lib:/opt/intel/oneapi/dal/latest/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/ccl/latest/lib/ \ - LD_LIBRARY_PATH=/opt/intel/oneapi/tbb/latest/env/../lib/intel64/gcc4.8:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib:/opt/intel/oneapi/itac/latest/slib:/opt/intel/oneapi/ippcp/latest/lib/:/opt/intel/oneapi/ipp/latest/lib:/opt/intel/oneapi/dpl/latest/lib:/opt/intel/oneapi/dnnl/latest/lib:/opt/intel/oneapi/debugger/latest/opt/debugger/lib:/opt/intel/oneapi/dal/latest/lib:/opt/intel/oneapi/compiler/latest/opt/oclfpga/host/linux64/lib:/opt/intel/oneapi/compiler/latest/opt/compiler/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/ccl/latest/lib/ \ - PATH=/opt/intel/oneapi/vtune/latest/bin64:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/bin:/opt/intel/oneapi/mpi/latest/bin:/opt/intel/oneapi/mkl/latest/bin/:/opt/intel/oneapi/itac/latest/bin:/opt/intel/oneapi/inspector/latest/bin64:/opt/intel/oneapi/dpcpp-ct/latest/bin:/opt/intel/oneapi/dev-utilities/latest/bin:/opt/intel/oneapi/debugger/latest/opt/debugger/bin:/opt/intel/oneapi/compiler/latest/opt/oclfpga/bin:/opt/intel/oneapi/compiler/latest/bin:/opt/intel/oneapi/advisor/latest/bin64:/opt/mamba/bin:/opt/mamba/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - MKLROOT=/opt/intel/oneapi/mkl/latest \ - FI_PROVIDER_PATH=/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib/prov:/usr/lib/x86_64-linux-gnu/libfabric \ - CMAKE_PREFIX_PATH=/opt/intel/oneapi/tbb/latest/env/..:/opt/intel/oneapi/mkl/latest/lib/cmake:/opt/intel/oneapi/ipp/latest/lib/cmake/ipp:/opt/intel/oneapi/dpl/latest/lib/cmake/oneDPL:/opt/intel/oneapi/dnnl/latest/lib/cmake:/opt/intel/oneapi/dal/latest:/opt/intel/oneapi/compiler/latest \ - CMPLR_ROOT=/opt/intel/oneapi/compiler/latest + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN ls -la /opt/intel/oneapi/mkl/latest/lib/intel64/*sycl*dft* +RUN ls -la /opt/intel/oneapi/mkl/latest/lib/intel64/*dis* + +# Set oneAPI environment variables +ENV ONEAPI_ROOT=/opt/intel/oneapi +ENV I_MPI_ROOT=${ONEAPI_ROOT}/mpi/latest +ENV MKLROOT=${ONEAPI_ROOT}/mkl/latest +ENV CMPLR_ROOT=${ONEAPI_ROOT}/compiler/latest + +# Set library paths and include paths +ENV LIBRARY_PATH=${ONEAPI_ROOT}/tbb/latest/lib/intel64/gcc4.8:${ONEAPI_ROOT}/mpi/latest/lib:${MKLROOT}/lib/intel64:${ONEAPI_ROOT}/compiler/latest/lib/:${LIBRARY_PATH} +ENV LD_LIBRARY_PATH=${ONEAPI_ROOT}/tbb/latest/lib/intel64/gcc4.8:${ONEAPI_ROOT}/mpi/latest/lib:${MKLROOT}/lib/intel64:${ONEAPI_ROOT}/compiler/latest/lib/:${LD_LIBRARY_PATH} +ENV PATH=${ONEAPI_ROOT}/vtune/latest/bin64:${ONEAPI_ROOT}/mpi/latest/bin:${MKLROOT}/bin/intel64:${ONEAPI_ROOT}/compiler/latest/bin:${PATH} +ENV CPATH=${MKLROOT}/include:${ONEAPI_ROOT}/mpi/latest/include:${CPATH} +ENV PKG_CONFIG_PATH=${MKLROOT}/lib/pkgconfig:${ONEAPI_ROOT}/mpi/latest/lib/pkgconfig:${PKG_CONFIG_PATH} + +# Set CMAKE related paths +ENV CMAKE_PREFIX_PATH=${ONEAPI_ROOT}/tbb/latest:${MKLROOT}/lib/cmake:${ONEAPI_ROOT}/dpl/latest/lib/cmake:${ONEAPI_ROOT}/dnnl/latest/lib/cmake:${ONEAPI_ROOT}/dal/latest:${ONEAPI_ROOT}/compiler/latest:${CMAKE_PREFIX_PATH} SHELL ["/bin/bash", "-c"] ENV CC=mpiicx CXX=mpiicpx FC=mpiifx +# Verify oneAPI installation +RUN source ${ONEAPI_ROOT}/setvars.sh && \ + echo "=== Verify compiler ===" && \ + which mpiicx && mpiicx --version && \ + echo "=== Verify MKL ===" && \ + ls ${MKLROOT}/lib/intel64/ && \ + echo "=== Verify MPI ===" && \ + which mpirun && mpirun --version + # https://elpa.mpcdf.mpg.de/software/tarball-archive/ELPA_TARBALL_ARCHIVE.html RUN source /opt/intel/oneapi/setvars.sh && \ cd /tmp && \ @@ -39,28 +63,34 @@ RUN source /opt/intel/oneapi/setvars.sh && \ tar xzf elpa-$ELPA_VER.tar.gz && rm elpa-$ELPA_VER.tar.gz && \ cd elpa-$ELPA_VER && mkdir build && cd build && \ ../configure CFLAGS="-O3 -march=native" FCFLAGS="-O3 -qmkl=cluster" --enable-openmp && \ - make -j`nproc` && \ + make -j$(nproc) && \ make PREFIX=/usr/local install && \ ln -s /usr/local/include/elpa_openmp-$ELPA_VER/elpa /usr/local/include/ && \ cd /tmp && rm -rf elpa-$ELPA_VER -RUN cd /tmp && git clone https://github.com/Tencent/rapidjson.git && cp -r rapidjson/include/rapidjson /usr/include/ \ - && rm -rf rapidjson - -RUN wget https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip \ - --no-check-certificate --quiet -O libtorch.zip && \ - unzip -q libtorch.zip -d /opt && rm libtorch.zip - -ENV CMAKE_PREFIX_PATH=/opt/libtorch/share/cmake +# rapidjson and libtorch +RUN cd /tmp && git clone --depth 1 https://github.com/Tencent/rapidjson.git && cp -r rapidjson/include/rapidjson /usr/include/ && rm -rf rapidjson +RUN wget -q https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-2.0.0%2Bcpu.zip -O /tmp/libtorch.zip && \ + unzip -q /tmp/libtorch.zip -d /opt && rm -f /tmp/libtorch.zip +ENV CMAKE_PREFIX_PATH=/opt/libtorch/share/cmake:${CMAKE_PREFIX_PATH} +# Clone and build abacus (optional during image build; keep for CI image) ADD https://api.github.com/repos/deepmodeling/abacus-develop/git/refs/heads/develop /dev/null - -RUN source /opt/intel/oneapi/setvars.sh && \ - git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ +RUN export LD_LIBRARY_PATH=${GKLIB_ROOT}/lib:${METIS32_ROOT}/lib:${PARMETIS32_ROOT}/lib:${SUPERLU32_DIST_ROOT}/lib:${PEXSI32_ROOT}/lib:${LD_LIBRARY_PATH} && \ + export CPATH=${GKLIB_ROOT}/include:${METIS32_ROOT}/include:${PARMETIS32_ROOT}/include:${SUPERLU32_DIST_ROOT}/include:${PEXSI32_ROOT}/include:${CPATH} && \ + cd /tmp && git clone https://github.com/deepmodeling/abacus-develop.git --depth 1 && \ cd abacus-develop && \ - cmake -B build -DENABLE_MLALGO=ON -DENABLE_LIBXC=ON -DENABLE_LIBRI=ON -DENABLE_RAPIDJSON=ON && \ - cmake --build build -j`nproc` && \ + cmake -B build \ + -DENABLE_MLALGO=ON \ + -DENABLE_LIBXC=ON \ + -DENABLE_LIBRI=ON \ + -DENABLE_RAPIDJSON=ON \ + -DCMAKE_BUILD_TYPE=Release && \ + cmake --build build -j"$(nproc)" && \ cmake --install build && \ - rm -rf build && \ - abacus --version - #&& rm -rf abacus-develop + (/usr/local/bin/abacus --version || echo "ABACUS installed but version check failed") && \ + rm -rf /tmp/abacus-develop + # -DMKL_SYCL=OFF \ + # -DMKL_SYCL_DISTRIBUTED_DFT:BOOL=OFF \ +# Default entry +CMD ["/bin/bash"] diff --git a/cmake/FindMKL.cmake b/cmake/FindMKL.cmake index f9cddfcf74..3680b8a92c 100644 --- a/cmake/FindMKL.cmake +++ b/cmake/FindMKL.cmake @@ -4,7 +4,7 @@ # MKL_INCLUDE - where to find mkl.h, etc. # MKL_FOUND - True if mkl found. -find_package(MKL NO_MODULE) # try using official module first +# find_package(MKL NO_MODULE) # try using official module first if(NOT TARGET MKL::MKL) find_path(MKL_INCLUDE mkl_service.h HINTS ${MKLROOT}/include) @@ -70,7 +70,9 @@ if(MKL_FOUND) set_property(TARGET MKL::MKL PROPERTY INTERFACE_LINK_LIBRARIES "-Wl,--start-group" - MKL::INTEL MKL::INTEL_THREAD MKL::CORE) + MKL::INTEL MKL::INTEL_THREAD MKL::CORE + "-Wl,--end-group" + ) endif() endif()