1.1.0 release candidate

.github/workflows/actions.yml

@@ -3,33 +3,42 @@ name: Unit tests
 on:
   pull_request:
     branches:
-      - master
+      - '**rc'
+      - 'master'
 
 jobs:
   tests:
     strategy:
+      fail-fast: false
       matrix:
-        cuda: ["ON", "OFF"]
-        precision: ["double", "single"]
-    runs-on:
-      - self-hosted
+        device: [amd-gpu, nvidia-gpu]
+        precision: [double, single]
+        exclude: 
+        - device: amd-gpu 
+          precision: double
+        # my AMD GPUs doesn't support fp64 atomics : (
+    runs-on: [self-hosted, "${{ matrix.device }}"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3.3.0
-      - name: Load CUDA Entity
-        if: ${{ matrix.cuda == 'ON' }}
+      - name: Configure
         run: |
-          module load entity/cuda/volta70/skx
-      - name: Load Entity without CUDA
-        if: ${{ matrix.cuda == 'OFF' }}
-        run: |
-          module load entity/skx
+          if [ "${{ matrix.device }}" = "nvidia-gpu" ]; then
+            FLAGS="-D Kokkos_ENABLE_CUDA=ON"
+            if [[ ! -z $(nvidia-smi | grep "V100") ]]; then
+              FLAGS+=" -D Kokkos_ARCH_VOLTA70=ON"
+            elif [[ ! -z $(nvidia-smi | grep "A100") ]]; then
+              FLAGS+=" -D Kokkos_ARCH_AMPERE80=ON"
+            else
+              FLAGS+=" -D Kokkos_ARCH_AMPERE86=ON"
+            fi
+          elif [ "${{ matrix.device }}" = "amd-gpu" ]; then
+            FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON"
+          fi
+          cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS
       - name: Compile
         run: |
-          cmake -B build -D TESTS=ON -D precision=${{ matrix.precision }}
-          cd build
-          make -j $(exec nproc)
+          cmake --build build -j $(nproc)
       - name: Run tests
         run: |
-          cd build/tests
-          ctest
+          ctest --test-dir build --output-on-failure --verbose

.gitignore

@@ -53,3 +53,4 @@ Testing/
 
 .schema.json
 *_old/
+action-token

CMakeLists.txt

@@ -5,7 +5,7 @@ set(PROJECT_NAME entity)
 
 project(
   ${PROJECT_NAME}
-  VERSION 1.0.0
+  VERSION 1.1.0
   LANGUAGES CXX C)
 add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
 execute_process(COMMAND

README.md

@@ -27,3 +27,7 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth
 🤷 __Arno Vanthieghem__ {[@vanthieg](https://github.com/vanthieg): framework, PIC}
 
 😺 __Muni Zhou__ {[@munizhou](https://github.com/munizhou): PIC}
+
+## Branch policy
+
+Master branch contains the latest stable version of the code which has already been released. Development on the core is done on branches starting with `dev/`, while fixing bugs is done in branches that start with `bug/`. User-specific modifications (i.e., new problem generators plus perhaps minor corrections in the core) are done on branches starting with `pgen/`. Before merging to the master branch, all the branches must first be merged to the latest release-candidate branch, which ends with `rc`, via a pull request. After which, when all the release goals are met, the `rc` branch is merged to the master and released as a new stable version. Stale branches will be archived with a tag starting with `archive/` (can still be accessed via the "Tags" tab) and removed.

cmake/report.cmake

@@ -150,6 +150,16 @@ PrintChoices("CUDA"
   0
   42
 )
+PrintChoices("HIP"
+  "Kokkos_ENABLE_HIP"
+  "${ON_OFF_VALUES}"
+  ${Kokkos_ENABLE_HIP}
+  "OFF"
+  "${Green}"
+  HIP_REPORT
+  0
+  42
+)
 PrintChoices("OpenMP"
   "Kokkos_ENABLE_OPENMP"
   "${ON_OFF_VALUES}"
@@ -183,6 +193,28 @@ PrintChoices("C compiler"
   42
 )
 
+get_cmake_property(_variableNames VARIABLES)
+foreach (_variableName ${_variableNames})
+    string(REGEX MATCH "Kokkos_ARCH_*" _isMatched ${_variableName})
+    if(_isMatched)
+        get_property(isSet CACHE ${_variableName} PROPERTY VALUE)
+        if(isSet STREQUAL "ON")
+          string(REGEX REPLACE "Kokkos_ARCH_" "" ARCH ${_variableName})
+          break()
+        endif()
+    endif()
+endforeach()
+PrintChoices("Architecture"
+  "Kokkos_ARCH_*"
+  "${ARCH}"
+  "${ARCH}"
+  "N/A"
+  "${ColorReset}"
+  ARCH_REPORT
+  0
+  42
+)
+
 if(${Kokkos_ENABLE_CUDA})
   if("${CMAKE_CUDA_COMPILER}" STREQUAL "")
     execute_process(COMMAND which nvcc OUTPUT_VARIABLE CUDACOMP)
@@ -194,14 +226,13 @@ if(${Kokkos_ENABLE_CUDA})
 
   message(STATUS "CUDA compiler: ${CUDACOMP}")
   execute_process(COMMAND bash -c "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'"
-
     OUTPUT_VARIABLE CUDACOMP_VERSION
     OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   PrintChoices("CUDA compiler"
     "CMAKE_CUDA_COMPILER"
-    "${CUDACOMP} v${CUDACOMP_VERSION}"
-    "${CUDACOMP} v${CUDACOMP_VERSION}"
+    "${CUDACOMP}"
+    "${CUDACOMP}"
     "N/A"
     "${ColorReset}"
     CUDA_COMPILER_REPORT
@@ -210,6 +241,12 @@ if(${Kokkos_ENABLE_CUDA})
   )
 endif()
 
+if (${Kokkos_ENABLE_HIP})
+  execute_process(COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '"
+    OUTPUT_VARIABLE ROCM_VERSION
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+endif()
+
 set(DOT_SYMBOL "${ColorReset}.")
 set(DOTTED_LINE_SYMBOL "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . ")
 
@@ -242,7 +279,9 @@ message("  ${OUTPUT_REPORT}")
 message("${DASHED_LINE_SYMBOL}
 Compile configurations")
 
+message("  ${ARCH_REPORT}")
 message("  ${CUDA_REPORT}")
+message("  ${HIP_REPORT}")
 message("  ${OPENMP_REPORT}")
 
 message("  ${C_COMPILER_REPORT}")
@@ -259,6 +298,11 @@ message("  ${DEBUG_REPORT}")
 
 message("${DASHED_LINE_SYMBOL}\nDependencies")
 
+if (NOT "${CUDACOMP_VERSION}" STREQUAL "")
+  message("  - CUDA:\tv${CUDACOMP_VERSION}")
+elseif(NOT "${ROCM_VERSION}" STREQUAL "")
+  message("  - ROCm:\tv${ROCM_VERSION}")
+endif()
 message("  - Kokkos:\tv${Kokkos_VERSION}")
 if(${output})
   message("  - ADIOS2:\tv${adios2_VERSION}")

dev/Dockerfile.common

@@ -0,0 +1,81 @@
+ARG DEBIAN_FRONTEND=noninteractive
+ENV DISPLAY=host.docker.internal:0.0
+
+# upgrade
+RUN apt-get update && apt-get upgrade -y
+
+# cmake & build tools
+RUN apt-get remove -y --purge cmake && \
+    apt-get install -y wget build-essential && \
+    wget "https://github.com/Kitware/CMake/releases/download/v3.29.6/cmake-3.29.6-linux-x86_64.tar.gz" -P /opt && \
+    tar xvf /opt/cmake-3.29.6-linux-x86_64.tar.gz -C /opt && \
+    rm /opt/cmake-3.29.6-linux-x86_64.tar.gz
+ENV PATH=/opt/cmake-3.29.6-linux-x86_64/bin:$PATH
+
+# adios2
+RUN apt-get update && apt-get install -y git libhdf5-dev && \
+    git clone https://github.com/ornladios/ADIOS2.git /opt/adios2-src && \
+    cd /opt/adios2-src && \
+    cmake -B build \
+      -D CMAKE_CXX_STANDARD=17 \
+      -D CMAKE_CXX_EXTENSIONS=OFF \
+      -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \
+      -D BUILD_SHARED_LIBS=ON \
+      -D ADIOS2_USE_HDF5=ON \
+      -D ADIOS2_USE_Python=OFF \
+      -D ADIOS2_USE_Fortran=OFF \
+      -D ADIOS2_USE_ZeroMQ=OFF \
+      -D BUILD_TESTING=OFF \
+      -D ADIOS2_BUILD_EXAMPLES=OFF \
+      -D ADIOS2_USE_MPI=OFF \
+      -D ADIOS2_HAVE_HDF5_VOL=OFF \
+      -D CMAKE_INSTALL_PREFIX=/opt/adios2 && \
+    cmake --build build -j && \
+    cmake --install build && \
+    rm -rf /opt/adios2-src
+ENV ADIOS2_DIR=/opt/adios2
+ENV HDF5_ROOT=/usr
+
+# other packages + python
+RUN apt-get update && \
+    apt-get install -y bc gpg curl ssh vim emacs bat fzf ffmpeg hdf5-tools software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && apt-get install -y python3.12-dev python3.12-venv
+
+# eza
+RUN mkdir -p /etc/apt/keyrings && \
+    wget -qO- https://raw.githubusercontent.com/eza-community/eza/main/deb.asc | gpg --dearmor -o /etc/apt/keyrings/gierens.gpg && \
+    echo "deb [signed-by=/etc/apt/keyrings/gierens.gpg] http://deb.gierens.de stable main" | tee /etc/apt/sources.list.d/gierens.list && \
+    chmod 644 /etc/apt/keyrings/gierens.gpg /etc/apt/sources.list.d/gierens.list && \
+    apt-get update && apt-get install -y eza
+
+# cleanup
+RUN apt-get clean && \
+    apt-get autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/cache/* && \
+    rm -rf /var/lib/log/* && \
+    rm -rf /var/lib/apt/lists/*
+
+# python
+RUN ln -s $(which python3.12) /bin/python && \
+    python -m venv /opt/venv && \
+    /opt/venv/bin/pip install --upgrade pip && \
+    /opt/venv/bin/pip install black numpy myplotlib nt2py jupyterlab ipykernel
+ENV VIRTUAL_ENV=/opt/venv
+
+# user environment
+ARG HOME=/root
+WORKDIR $HOME
+RUN echo "alias ls='eza -a --sort=type'" >> $HOME/.bashrc && \
+    echo "alias ll='eza -a --long --header --sort=type --time-style=long-iso'" >> $HOME/.bashrc && \
+    echo "alias cat='batcat -pp'" >> $HOME/.bashrc
+
+RUN curl -sS https://starship.rs/install.sh | sh -s -- --yes && \
+    echo eval "\$(starship init bash)" >> $HOME/.bashrc && \
+    mkdir -p $HOME/.config && \
+    starship preset pure-preset -o $HOME/.config/starship.toml && \
+    sed -i 's/$python\\//g' $HOME/.config/starship.toml
+ENV STARSHIP_CONFIG=$HOME/.config/starship.toml
+
+ENV PATH=/opt/adios2/bin:/opt/venv/bin:$PATH