Pgen-specific custom field output

.github/workflows/actions.yml

@@ -11,23 +11,34 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        gpu: ["amd", "nvidia"]
-        precision: ["double", "single"]
-    runs-on: [self-hosted, "${{ matrix.gpu }}-gpu"]
+        device: [amd-gpu, nvidia-gpu]
+        precision: [double, single]
+        exclude: 
+        - device: amd-gpu 
+          precision: double
+        # my AMD GPUs doesn't support fp64 atomics : (
+    runs-on: [self-hosted, "${{ matrix.device }}"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3.3.0
       - name: Configure
         run: |
-          if [ "${{ matrix.gpu }}" = "nvidia" ]; then
-            FLAGS="-D Kokkos_ENABLE_CUDA=ON -D Kokkos_ARCH_AMPERE86=ON"
-          elif [ "${{ matrix.gpu }}" = "amd" ]; then
+          if [ "${{ matrix.device }}" = "nvidia-gpu" ]; then
+            FLAGS="-D Kokkos_ENABLE_CUDA=ON"
+            if [[ ! -z $(nvidia-smi | grep "V100") ]]; then
+              FLAGS+=" -D Kokkos_ARCH_VOLTA70=ON"
+            elif [[ ! -z $(nvidia-smi | grep "A100") ]]; then
+              FLAGS+=" -D Kokkos_ARCH_AMPERE80=ON"
+            else
+              FLAGS+=" -D Kokkos_ARCH_AMPERE86=ON"
+            fi
+          elif [ "${{ matrix.device }}" = "amd-gpu" ]; then
             FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON"
           fi
           cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS
       - name: Compile
         run: |
-          cmake --build build -j $(exec nproc)
+          cmake --build build -j $(nproc)
       - name: Run tests
         run: |
-          ctest --test-dir build --output-on-failure --verbose
+          ctest --test-dir build --output-on-failure --verbose

.gitignore

@@ -53,3 +53,4 @@ Testing/
 
 .schema.json
 *_old/
+action-token

cmake/report.cmake

@@ -150,6 +150,16 @@ PrintChoices("CUDA"
   0
   42
 )
+PrintChoices("HIP"
+  "Kokkos_ENABLE_HIP"
+  "${ON_OFF_VALUES}"
+  ${Kokkos_ENABLE_HIP}
+  "OFF"
+  "${Green}"
+  HIP_REPORT
+  0
+  42
+)
 PrintChoices("OpenMP"
   "Kokkos_ENABLE_OPENMP"
   "${ON_OFF_VALUES}"
@@ -183,6 +193,28 @@ PrintChoices("C compiler"
   42
 )
 
+get_cmake_property(_variableNames VARIABLES)
+foreach (_variableName ${_variableNames})
+    string(REGEX MATCH "Kokkos_ARCH_*" _isMatched ${_variableName})
+    if(_isMatched)
+        get_property(isSet CACHE ${_variableName} PROPERTY VALUE)
+        if(isSet STREQUAL "ON")
+          string(REGEX REPLACE "Kokkos_ARCH_" "" ARCH ${_variableName})
+          break()
+        endif()
+    endif()
+endforeach()
+PrintChoices("Architecture"
+  "Kokkos_ARCH_*"
+  "${ARCH}"
+  "${ARCH}"
+  "N/A"
+  "${ColorReset}"
+  ARCH_REPORT
+  0
+  42
+)
+
 if(${Kokkos_ENABLE_CUDA})
   if("${CMAKE_CUDA_COMPILER}" STREQUAL "")
     execute_process(COMMAND which nvcc OUTPUT_VARIABLE CUDACOMP)
@@ -194,14 +226,13 @@ if(${Kokkos_ENABLE_CUDA})
 
   message(STATUS "CUDA compiler: ${CUDACOMP}")
   execute_process(COMMAND bash -c "${CUDACOMP} --version | grep release | sed -e 's/.*release //' -e 's/,.*//'"
-
     OUTPUT_VARIABLE CUDACOMP_VERSION
     OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   PrintChoices("CUDA compiler"
     "CMAKE_CUDA_COMPILER"
-    "${CUDACOMP} v${CUDACOMP_VERSION}"
-    "${CUDACOMP} v${CUDACOMP_VERSION}"
+    "${CUDACOMP}"
+    "${CUDACOMP}"
     "N/A"
     "${ColorReset}"
     CUDA_COMPILER_REPORT
@@ -210,6 +241,12 @@ if(${Kokkos_ENABLE_CUDA})
   )
 endif()
 
+if (${Kokkos_ENABLE_HIP})
+  execute_process(COMMAND bash -c "hipcc --version | grep HIP | cut -d ':' -f 2 | tr -d ' '"
+    OUTPUT_VARIABLE ROCM_VERSION
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+endif()
+
 set(DOT_SYMBOL "${ColorReset}.")
 set(DOTTED_LINE_SYMBOL "${ColorReset}. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . ")
 
@@ -242,7 +279,9 @@ message("  ${OUTPUT_REPORT}")
 message("${DASHED_LINE_SYMBOL}
 Compile configurations")
 
+message("  ${ARCH_REPORT}")
 message("  ${CUDA_REPORT}")
+message("  ${HIP_REPORT}")
 message("  ${OPENMP_REPORT}")
 
 message("  ${C_COMPILER_REPORT}")
@@ -259,6 +298,11 @@ message("  ${DEBUG_REPORT}")
 
 message("${DASHED_LINE_SYMBOL}\nDependencies")
 
+if (NOT "${CUDACOMP_VERSION}" STREQUAL "")
+  message("  - CUDA:\tv${CUDACOMP_VERSION}")
+elseif(NOT "${ROCM_VERSION}" STREQUAL "")
+  message("  - ROCm:\tv${ROCM_VERSION}")
+endif()
 message("  - Kokkos:\tv${Kokkos_VERSION}")
 if(${output})
   message("  - ADIOS2:\tv${adios2_VERSION}")

dev/runners/Dockerfile.runner.nvidia → dev/runners/Dockerfile.runner.cuda

@@ -59,14 +59,14 @@ ARG HOME=/home/$USER
 WORKDIR $HOME
 
 # gh runner
-ARG TOKEN
 RUN mkdir actions-runner
 WORKDIR $HOME/actions-runner
 
-RUN curl -o actions-runner-linux-x64-2.317.0.tar.gz \
+RUN --mount=type=secret,id=ghtoken \
+    curl -o actions-runner-linux-x64-2.317.0.tar.gz \
          -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-linux-x64-2.317.0.tar.gz && \
     tar xzf ./actions-runner-linux-x64-2.317.0.tar.gz && \
     sudo ./bin/installdependencies.sh && \
-    ./config.sh --url https://github.com/entity-toolkit/entity --token $TOKEN --labels nvidia-gpu
+    ./config.sh --url https://github.com/entity-toolkit/entity --token "$(sudo cat /run/secrets/ghtoken)" --labels nvidia-gpu
 
 ENTRYPOINT ["./run.sh"]

dev/runners/Dockerfile.runner.rocm

@@ -0,0 +1,88 @@
+FROM rocm/rocm-terminal:latest
+
+USER root
+ENV PATH=/opt/rocm/bin:$PATH
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV DISPLAY=host.docker.internal:0.0
+
+# upgrade
+RUN apt-get update && apt-get upgrade -y
+
+# cmake & build tools
+RUN apt-get remove -y --purge cmake && \
+    apt-get install -y sudo wget curl build-essential && \
+    wget "https://github.com/Kitware/CMake/releases/download/v3.29.6/cmake-3.29.6-linux-x86_64.tar.gz" -P /opt && \
+    tar xvf /opt/cmake-3.29.6-linux-x86_64.tar.gz -C /opt && \
+    rm /opt/cmake-3.29.6-linux-x86_64.tar.gz
+ENV PATH=/opt/cmake-3.29.6-linux-x86_64/bin:$PATH
+
+# adios2
+RUN apt-get update && apt-get install -y git libhdf5-dev && \
+    git clone https://github.com/ornladios/ADIOS2.git /opt/adios2-src && \
+    cd /opt/adios2-src && \
+    cmake -B build \
+      -D CMAKE_CXX_STANDARD=17 \
+      -D CMAKE_CXX_EXTENSIONS=OFF \
+      -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \
+      -D BUILD_SHARED_LIBS=ON \
+      -D ADIOS2_USE_HDF5=ON \
+      -D ADIOS2_USE_Python=OFF \
+      -D ADIOS2_USE_Fortran=OFF \
+      -D ADIOS2_USE_ZeroMQ=OFF \
+      -D BUILD_TESTING=OFF \
+      -D ADIOS2_BUILD_EXAMPLES=OFF \
+      -D ADIOS2_USE_MPI=OFF \
+      -D ADIOS2_HAVE_HDF5_VOL=OFF \
+      -D CMAKE_INSTALL_PREFIX=/opt/adios2 && \
+    cmake --build build -j && \
+    cmake --install build && \
+    rm -rf /opt/adios2-src
+
+ENV HDF5_ROOT=/usr
+ENV ADIOS2_DIR=/opt/adios2
+ENV PATH=/opt/adios2/bin:$PATH
+
+# additional ROCm packages
+RUN git clone -b release/rocm-rel-6.1.1.1 https://github.com/ROCm/rocThrust.git /opt/rocthrust-src && \
+    git clone -b release/rocm-rel-6.1.00.36 https://github.com/ROCm/rocPRIM.git /opt/rocprim-src && \
+    cd /opt/rocthrust-src && ./install --install && \
+    cd /opt/rocprim-src && ./install --install && \
+    rm -rf /opt/rocthrust-src /opt/rocprim-src
+
+ENV CMAKE_PREFIX_PATH=/opt/rocm 
+ENV CC=hipcc 
+ENV CXX=hipcc
+
+# cleanup
+RUN apt-get clean && \
+    apt-get autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/cache/* && \
+    rm -rf /var/lib/log/* && \
+    rm -rf /var/lib/apt/lists/*
+
+ARG USER=runner
+RUN useradd -ms /usr/bin/zsh $USER && \
+    usermod -aG sudo $USER && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER $USER
+ARG HOME=/home/$USER
+WORKDIR $HOME
+
+# gh runner
+ARG RUNNER_VERSION=2.317.0
+RUN mkdir actions-runner
+WORKDIR $HOME/actions-runner
+
+RUN curl -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+         -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    sudo ./bin/installdependencies.sh
+
+ADD start.sh start.sh
+RUN sudo chown $USER:$USER start.sh && \
+    sudo chmod +x start.sh
+
+ENTRYPOINT ["./start.sh"]

dev/runners/README.md

@@ -4,12 +4,18 @@ GitHub allows to listen to repository changes and run the so-called "actions" (e
 
 To do that, one needs to create an image with the corresponding `Dockerfile`, and then launch a Docker container which will run in the background, listening to commands and running any actions forwarded from the GitHub.
 
+First, you will need to obtain a runner token from the Entity GitHub repo, by going to Settings -> Actions -> Runners -> New self-hosted runner. Copy the token to use it later. The images differ slightly from the type of runner. 
+
 ### NVIDIA GPUs
 
 ```sh
-# 1. Create the image
-docker build -t ghrunner:nvidia -f Dockerfile.runner.nvidia .
-# 2. Run a container from the image with GPU support 
-# ... (see wiki for instructions on NVIDIA runtime)
-docker run --runtime=nvidia --gpus=all -dt ghrunner:nvidia
+docker build -t ghrunner:nvidia -f Dockerfile.runner.cuda .
+docker run -e TOKEN=<TOKEN> -e LABEL=nvidia-gpu --runtime=nvidia --gpus=all -dt ghrunner:nvidia
+```
+
+### AMD GPUs
+
+```sh
+docker build -t ghrunner:amd -f Dockerfile.runner.rocm .
+docker run -e TOKEN=<TOKEN> -e LABEL=amd-gpu --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video -dt ghrunner:amd
 ```

dev/runners/start.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+./config.sh --unattended --url https://github.com/entity-toolkit/entity --token ${TOKEN} --labels ${LABEL}
+
+if [[ ${LABEL} == "amd-gpu" ]]; then
+  echo "AMD GPU runner detected"
+  export HSA_OVERRIDE_GFX_VERSION=11.0.0 HIP_VISIBLE_DEVICES=0 ROCR_VISIBLE_DEVICES=0
+else
+  echo "Non-AMD runner"
+fi
+
+cleanup() {
+  echo "Removing runner..."
+  ./config.sh remove --unattended --token ${TOKEN}
+}
+
+trap 'cleanup; exit 130' INT
+trap 'cleanup; exit 143' TERM
+
+./run.sh &
+wait $!

dev/welcome.rocm

@@ -68,7 +68,8 @@ else
   python_version=${error_msg}
 fi
 
-str=$(cat <<EOF
+str=$(
+  cat <<EOF
 ${FgBlue}              __        __ 
              /\\ \\__  __/\\ \\__ 
     __    ___\\ \\  _\\/\\_\\ \\  _\\  __  __ 
@@ -79,8 +80,6 @@ ${FgBlue}              __        __
                                     /\\___/ 
 ${Reset}${Underscore}Entity ROCm container${Reset}${FgBlue}               \\/__/${Reset} 
 
-${FgRed}This image is experimental, and may not work out of the box.${Reset}
-
 ## Installed packages:
   - ${gcc_version}
   - ${rocm_version}
@@ -101,13 +100,16 @@ ${FgRed}This image is experimental, and may not work out of the box.${Reset}
 * Build the code:
   ${FgGreen}$ cmake --build build -j \$(nproc)${Reset}
 
+* Do not forget to set the following environment variables to their appropriate values:
+  ${FgGreen}$ HSA_OVERRIDE_GFX_VERSION=**.*.* HIP_VISIBLE_DEVICES=* ROCR_VISIBLE_DEVICES=*${RESET}
+
 * The executable will be generated as \`build/src/entity.xc\`. Run it by specifying the input file:
   ${FgGreen}$ build/src/entity.xc -input <input_file>${Reset}
 
 ## Testing the code
 
 * To build the tests, remove the \`build\` directory, and preconfigure with:
-  ${FgGreen}$ cmake -B build -D TESTS=ON -D output=ON -D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_***=ON${Reset}
+  ${FgGreen}$ cmake -B build -D TESTS=ON -D output=ON -D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_***=ON${Reset}
 
 * Then run the tests with:
   ${FgGreen}$ ctest --test-dir build${Reset}
@@ -117,5 +119,4 @@ ${Red}Enjoy!${Reset} 🚀
 EOF
 )
 
-
 printf "$str\n"

docker-compose.yml

@@ -30,7 +30,7 @@ services:
   entity-rocm:
     container_name: entity_rocm
     devices:
-      - "/dev/fdk"
+      - "/dev/kfd"
       - "/dev/dri"
     security_opt:
       - seccomp:unconfined

input.example.toml

@@ -336,6 +336,10 @@
     #   @note: For T, in cartesian can also use "x" "y" "z" instead of "1" "2" "3"
     #   @note: By default, we accumulate moments from all massive species, one can specify only specific species: e.g., Ttt_1_2, Rho_1, Rho_3_4
     quantities = ""
+    # Custom (user-defined) field quantities:
+    #   @type: array of strings
+    #   @default: []
+    custom = ""
     # @NOT_IMPLEMENTED: Stride for the output of fields:
     #   @type: unsigned short: > 1
     #   @default: 1