Potential issue in MPI comm for curvilinear SRPIC

.github/workflows/actions.yml

@@ -11,23 +11,34 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        gpu: ["amd", "nvidia"]
-        precision: ["double", "single"]
-    runs-on: [self-hosted, "${{ matrix.gpu }}-gpu"]
+        device: [amd-gpu, nvidia-gpu]
+        precision: [double, single]
+        exclude: 
+        - device: amd-gpu 
+          precision: double
+        # my AMD GPUs doesn't support fp64 atomics : (
+    runs-on: [self-hosted, "${{ matrix.device }}"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3.3.0
       - name: Configure
         run: |
-          if [ "${{ matrix.gpu }}" = "nvidia" ]; then
-            FLAGS="-D Kokkos_ENABLE_CUDA=ON -D Kokkos_ARCH_AMPERE86=ON"
-          elif [ "${{ matrix.gpu }}" = "amd" ]; then
+          if [ "${{ matrix.device }}" = "nvidia-gpu" ]; then
+            FLAGS="-D Kokkos_ENABLE_CUDA=ON"
+            if [[ ! -z $(nvidia-smi | grep "V100") ]]; then
+              FLAGS+=" -D Kokkos_ARCH_VOLTA70=ON"
+            elif [[ ! -z $(nvidia-smi | grep "A100") ]]; then
+              FLAGS+=" -D Kokkos_ARCH_AMPERE80=ON"
+            else
+              FLAGS+=" -D Kokkos_ARCH_AMPERE86=ON"
+            fi
+          elif [ "${{ matrix.device }}" = "amd-gpu" ]; then
             FLAGS="-D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_AMD_GFX1100=ON"
           fi
           cmake -B build -D TESTS=ON -D output=ON -D precision=${{ matrix.precision }} $FLAGS
       - name: Compile
         run: |
-          cmake --build build -j $(exec nproc)
+          cmake --build build -j $(nproc)
       - name: Run tests
         run: |
-          ctest --test-dir build --output-on-failure --verbose
+          ctest --test-dir build --output-on-failure --verbose

.gitignore

@@ -53,3 +53,4 @@ Testing/
 
 .schema.json
 *_old/
+action-token

dev/runners/Dockerfile.runner.nvidia → dev/runners/Dockerfile.runner.cuda

@@ -59,14 +59,14 @@ ARG HOME=/home/$USER
 WORKDIR $HOME
 
 # gh runner
-ARG TOKEN
 RUN mkdir actions-runner
 WORKDIR $HOME/actions-runner
 
-RUN curl -o actions-runner-linux-x64-2.317.0.tar.gz \
+RUN --mount=type=secret,id=ghtoken \
+    curl -o actions-runner-linux-x64-2.317.0.tar.gz \
          -L https://github.com/actions/runner/releases/download/v2.317.0/actions-runner-linux-x64-2.317.0.tar.gz && \
     tar xzf ./actions-runner-linux-x64-2.317.0.tar.gz && \
     sudo ./bin/installdependencies.sh && \
-    ./config.sh --url https://github.com/entity-toolkit/entity --token $TOKEN --labels nvidia-gpu
+    ./config.sh --url https://github.com/entity-toolkit/entity --token "$(sudo cat /run/secrets/ghtoken)" --labels nvidia-gpu
 
 ENTRYPOINT ["./run.sh"]

dev/runners/Dockerfile.runner.rocm

@@ -0,0 +1,88 @@
+FROM rocm/rocm-terminal:latest
+
+USER root
+ENV PATH=/opt/rocm/bin:$PATH
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV DISPLAY=host.docker.internal:0.0
+
+# upgrade
+RUN apt-get update && apt-get upgrade -y
+
+# cmake & build tools
+RUN apt-get remove -y --purge cmake && \
+    apt-get install -y sudo wget curl build-essential && \
+    wget "https://github.com/Kitware/CMake/releases/download/v3.29.6/cmake-3.29.6-linux-x86_64.tar.gz" -P /opt && \
+    tar xvf /opt/cmake-3.29.6-linux-x86_64.tar.gz -C /opt && \
+    rm /opt/cmake-3.29.6-linux-x86_64.tar.gz
+ENV PATH=/opt/cmake-3.29.6-linux-x86_64/bin:$PATH
+
+# adios2
+RUN apt-get update && apt-get install -y git libhdf5-dev && \
+    git clone https://github.com/ornladios/ADIOS2.git /opt/adios2-src && \
+    cd /opt/adios2-src && \
+    cmake -B build \
+      -D CMAKE_CXX_STANDARD=17 \
+      -D CMAKE_CXX_EXTENSIONS=OFF \
+      -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \
+      -D BUILD_SHARED_LIBS=ON \
+      -D ADIOS2_USE_HDF5=ON \
+      -D ADIOS2_USE_Python=OFF \
+      -D ADIOS2_USE_Fortran=OFF \
+      -D ADIOS2_USE_ZeroMQ=OFF \
+      -D BUILD_TESTING=OFF \
+      -D ADIOS2_BUILD_EXAMPLES=OFF \
+      -D ADIOS2_USE_MPI=OFF \
+      -D ADIOS2_HAVE_HDF5_VOL=OFF \
+      -D CMAKE_INSTALL_PREFIX=/opt/adios2 && \
+    cmake --build build -j && \
+    cmake --install build && \
+    rm -rf /opt/adios2-src
+
+ENV HDF5_ROOT=/usr
+ENV ADIOS2_DIR=/opt/adios2
+ENV PATH=/opt/adios2/bin:$PATH
+
+# additional ROCm packages
+RUN git clone -b release/rocm-rel-6.1.1.1 https://github.com/ROCm/rocThrust.git /opt/rocthrust-src && \
+    git clone -b release/rocm-rel-6.1.00.36 https://github.com/ROCm/rocPRIM.git /opt/rocprim-src && \
+    cd /opt/rocthrust-src && ./install --install && \
+    cd /opt/rocprim-src && ./install --install && \
+    rm -rf /opt/rocthrust-src /opt/rocprim-src
+
+ENV CMAKE_PREFIX_PATH=/opt/rocm 
+ENV CC=hipcc 
+ENV CXX=hipcc
+
+# cleanup
+RUN apt-get clean && \
+    apt-get autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/cache/* && \
+    rm -rf /var/lib/log/* && \
+    rm -rf /var/lib/apt/lists/*
+
+ARG USER=runner
+RUN useradd -ms /usr/bin/zsh $USER && \
+    usermod -aG sudo $USER && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER $USER
+ARG HOME=/home/$USER
+WORKDIR $HOME
+
+# gh runner
+ARG RUNNER_VERSION=2.317.0
+RUN mkdir actions-runner
+WORKDIR $HOME/actions-runner
+
+RUN curl -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+         -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    sudo ./bin/installdependencies.sh
+
+ADD start.sh start.sh
+RUN sudo chown $USER:$USER start.sh && \
+    sudo chmod +x start.sh
+
+ENTRYPOINT ["./start.sh"]

dev/runners/README.md

@@ -4,12 +4,18 @@ GitHub allows to listen to repository changes and run the so-called "actions" (e
 
 To do that, one needs to create an image with the corresponding `Dockerfile`, and then launch a Docker container which will run in the background, listening to commands and running any actions forwarded from the GitHub.
 
+First, you will need to obtain a runner token from the Entity GitHub repo, by going to Settings -> Actions -> Runners -> New self-hosted runner. Copy the token to use it later. The images differ slightly from the type of runner. 
+
 ### NVIDIA GPUs
 
 ```sh
-# 1. Create the image
-docker build -t ghrunner:nvidia -f Dockerfile.runner.nvidia .
-# 2. Run a container from the image with GPU support 
-# ... (see wiki for instructions on NVIDIA runtime)
-docker run --runtime=nvidia --gpus=all -dt ghrunner:nvidia
+docker build -t ghrunner:nvidia -f Dockerfile.runner.cuda .
+docker run -e TOKEN=<TOKEN> -e LABEL=nvidia-gpu --runtime=nvidia --gpus=all -dt ghrunner:nvidia
+```
+
+### AMD GPUs
+
+```sh
+docker build -t ghrunner:amd -f Dockerfile.runner.rocm .
+docker run -e TOKEN=<TOKEN> -e LABEL=amd-gpu --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video -dt ghrunner:amd
 ```

dev/runners/start.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+./config.sh --unattended --url https://github.com/entity-toolkit/entity --token ${TOKEN} --labels ${LABEL}
+
+if [[ ${LABEL} == "amd-gpu" ]]; then
+  echo "AMD GPU runner detected"
+  export HSA_OVERRIDE_GFX_VERSION=11.0.0 HIP_VISIBLE_DEVICES=0 ROCR_VISIBLE_DEVICES=0
+else
+  echo "Non-AMD runner"
+fi
+
+cleanup() {
+  echo "Removing runner..."
+  ./config.sh remove --unattended --token ${TOKEN}
+}
+
+trap 'cleanup; exit 130' INT
+trap 'cleanup; exit 143' TERM
+
+./run.sh &
+wait $!

docker-compose.yml

@@ -30,7 +30,7 @@ services:
   entity-rocm:
     container_name: entity_rocm
     devices:
-      - "/dev/fdk"
+      - "/dev/kfd"
       - "/dev/dri"
     security_opt:
       - seccomp:unconfined

src/CMakeLists.txt

@@ -37,4 +37,4 @@ add_subdirectory(${SRC_DIR}/../setups ${CMAKE_CURRENT_BINARY_DIR}/setups)
 
 set(libs ntt_global ntt_framework ntt_metrics ntt_engines ntt_pgen)
 add_dependencies(${ENTITY} ${libs})
-target_link_libraries(${ENTITY} PUBLIC ${libs})
+target_link_libraries(${ENTITY} PUBLIC ${libs})

src/archetypes/particle_injector.h

@@ -1,13 +1,17 @@
 /**
  * @file archetypes/particle_injector.h
- * @brief Particle injector routines
- * ...
+ * @brief Particle injector routines and classes
+ * @implements
+ *   - arch::UniformInjector<>
+ *   - arch::NonUniformInjector<>
+ *   - arch::AtmosphereInjector<>
+ *   - arch::InjectUniform<> -> void
+ *   - arch::InjectGlobally<> -> void
+ *   - arch::InjectNonUniform<> -> void
+ * @namespaces:
+ *   - arch::
  */
 
-/* -------------------------------------------------------------------------- */
-/* This header file is still under construction                               */
-/* -------------------------------------------------------------------------- */
-
 #ifndef ARCHETYPES_PARTICLE_INJECTOR_H
 #define ARCHETYPES_PARTICLE_INJECTOR_H
 

src/engines/engine_printer.cpp

@@ -137,6 +137,7 @@ namespace ntt {
         const auto hash           = std::string(ENTITY_GIT_HASH);
         const auto pgen           = std::string(PGEN);
         const auto nspec          = metadomain.species_params().size();
+        const auto precision      = (sizeof(real_t) == 4) ? "single" : "double";
 
 #if defined(__clang__)
         const std::string ccx = "Clang/LLVM " __clang_version__;
@@ -219,6 +220,7 @@ namespace ntt {
         add_param(report, 4, "HDF5", "%s", hdf5_version.c_str());
         add_param(report, 4, "Kokkos", "%s", kokkos_version.c_str());
         add_param(report, 4, "ADIOS2", "%s", adios2_version.c_str());
+        add_param(report, 4, "Precision", "%s", precision);
         add_param(report, 4, "Debug", "%s", dbg.c_str());
         report += "\n";
         add_category(report, 4, "Configuration");

src/engines/srpic.hpp

@@ -208,6 +208,7 @@ namespace ntt {
                       m_params.template get<real_t>(
                         "algorithms.timestep.correction") *
                       dt;
+      auto range = range_with_axis_BCs(domain);
       if constexpr (M::CoordType == Coord::Cart) {
         // minkowski case
         const auto dx = math::sqrt(domain.mesh.metric.template h_<1, 1>({}));
@@ -222,15 +223,9 @@ namespace ntt {
 
         Kokkos::parallel_for(
           "Ampere",
-          domain.mesh.rangeActiveCells(),
+          range,
           kernel::mink::Ampere_kernel<M::Dim>(domain.fields.em, coeff1, coeff2));
       } else {
-        range_t<M::Dim> range {};
-        if constexpr (M::Dim == Dim::_2D) {
-          range = CreateRangePolicy<Dim::_2D>(
-            { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) },
-            { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 });
-        }
         const auto ni2 = domain.mesh.n_active(in::x2);
         Kokkos::parallel_for("Ampere",
                              range,
@@ -538,13 +533,8 @@ namespace ntt {
                                                       coeff / V0,
                                                       ONE / n0));
       } else {
-        range_t<M::Dim> range {};
-        if constexpr (M::Dim == Dim::_2D) {
-          range = CreateRangePolicy<Dim::_2D>(
-            { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) },
-            { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 });
-        }
-        const auto ni2 = domain.mesh.n_active(in::x2);
+        auto       range = range_with_axis_BCs(domain);
+        const auto ni2   = domain.mesh.n_active(in::x2);
         Kokkos::parallel_for(
           "Ampere",
           range,
@@ -560,34 +550,7 @@ namespace ntt {
 
     void CurrentsFilter(domain_t& domain) {
       logger::Checkpoint("Launching currents filtering kernels", HERE);
-      range_t<M::Dim> range = domain.mesh.rangeActiveCells();
-      if constexpr (M::CoordType != Coord::Cart) {
-        /**
-         * @brief taking one extra cell in the x2 direction if AXIS BCs
-         *    . . . . .
-         *    . ^= =^ .
-         *    . |* *\*.
-         *    . |* *\*.
-         *    . ^- -^ .
-         *    . . . . .
-         */
-        if constexpr (M::Dim == Dim::_2D) {
-          if (domain.mesh.flds_bc_in({ +1, 0 }) == FldsBC::AXIS) {
-            range = CreateRangePolicy<Dim::_2D>(
-              { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) },
-              { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 });
-          }
-        } else if constexpr (M::Dim == Dim::_3D) {
-          if (domain.mesh.flds_bc_in({ +1, 0, 0 }) == FldsBC::AXIS) {
-            range = CreateRangePolicy<Dim::_3D>({ domain.mesh.i_min(in::x1),
-                                                  domain.mesh.i_min(in::x2),
-                                                  domain.mesh.i_min(in::x3) },
-                                                { domain.mesh.i_max(in::x1),
-                                                  domain.mesh.i_max(in::x2) + 1,
-                                                  domain.mesh.i_max(in::x3) });
-          }
-        }
-      }
+      auto       range   = range_with_axis_BCs(domain);
       const auto nfilter = m_params.template get<unsigned short>(
         "algorithms.current_filters");
       tuple_t<std::size_t, M::Dim> size;
@@ -1228,6 +1191,32 @@ namespace ntt {
       }
       return { sign, dim, xg_min, xg_max };
     }
+
+    auto range_with_axis_BCs(const domain_t& domain) -> range_t<M::Dim> {
+      auto range = domain.mesh.rangeActiveCells();
+      if constexpr (M::CoordType != Coord::Cart) {
+        /**
+         * @brief taking one extra cell in the x2 direction if AXIS BCs
+         */
+        if constexpr (M::Dim == Dim::_2D) {
+          if (domain.mesh.flds_bc_in({ 0, +1 }) == FldsBC::AXIS) {
+            range = CreateRangePolicy<Dim::_2D>(
+              { domain.mesh.i_min(in::x1), domain.mesh.i_min(in::x2) },
+              { domain.mesh.i_max(in::x1), domain.mesh.i_max(in::x2) + 1 });
+          }
+        } else if constexpr (M::Dim == Dim::_3D) {
+          if (domain.mesh.flds_bc_in({ 0, +1, 0 }) == FldsBC::AXIS) {
+            range = CreateRangePolicy<Dim::_3D>({ domain.mesh.i_min(in::x1),
+                                                  domain.mesh.i_min(in::x2),
+                                                  domain.mesh.i_min(in::x3) },
+                                                { domain.mesh.i_max(in::x1),
+                                                  domain.mesh.i_max(in::x2) + 1,
+                                                  domain.mesh.i_max(in::x3) });
+          }
+        }
+      }
+      return range;
+    }
   };
 
 } // namespace ntt