Branch policy added to README

README.md

@@ -27,3 +27,7 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth
 🤷 __Arno Vanthieghem__ {[@vanthieg](https://github.com/vanthieg): framework, PIC}
 
 😺 __Muni Zhou__ {[@munizhou](https://github.com/munizhou): PIC}
+
+## Branch policy
+
+Master branch contains the latest stable version of the code which has already been released. Development on the core is done on branches starting with `dev/`. User-specific modifications (i.e., new problem generators plus perhaps minor corrections in the core) are done on branches starting with `pgen/`. Before merging to the master branch, all the branches must first be merged to the latest release-candidate branch, which ends with `rc`, via a pull request. After which, when all the release goals are met, the `rc` branch is merged to the master and released as a new stable version. Stale branches will be archived with a tag starting with `archive/` (can still be accessed via the "Tags" tab) and removed.

dev/Dockerfile.common

@@ -0,0 +1,81 @@
+ARG DEBIAN_FRONTEND=noninteractive
+ENV DISPLAY=host.docker.internal:0.0
+
+# upgrade
+RUN apt-get update && apt-get upgrade -y
+
+# cmake & build tools
+RUN apt-get remove -y --purge cmake && \
+    apt-get install -y wget build-essential && \
+    wget "https://github.com/Kitware/CMake/releases/download/v3.29.6/cmake-3.29.6-linux-x86_64.tar.gz" -P /opt && \
+    tar xvf /opt/cmake-3.29.6-linux-x86_64.tar.gz -C /opt && \
+    rm /opt/cmake-3.29.6-linux-x86_64.tar.gz
+ENV PATH=/opt/cmake-3.29.6-linux-x86_64/bin:$PATH
+
+# adios2
+RUN apt-get update && apt-get install -y git libhdf5-dev && \
+    git clone https://github.com/ornladios/ADIOS2.git /opt/adios2-src && \
+    cd /opt/adios2-src && \
+    cmake -B build \
+      -D CMAKE_CXX_STANDARD=17 \
+      -D CMAKE_CXX_EXTENSIONS=OFF \
+      -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \
+      -D BUILD_SHARED_LIBS=ON \
+      -D ADIOS2_USE_HDF5=ON \
+      -D ADIOS2_USE_Python=OFF \
+      -D ADIOS2_USE_Fortran=OFF \
+      -D ADIOS2_USE_ZeroMQ=OFF \
+      -D BUILD_TESTING=OFF \
+      -D ADIOS2_BUILD_EXAMPLES=OFF \
+      -D ADIOS2_USE_MPI=OFF \
+      -D ADIOS2_HAVE_HDF5_VOL=OFF \
+      -D CMAKE_INSTALL_PREFIX=/opt/adios2 && \
+    cmake --build build -j && \
+    cmake --install build && \
+    rm -rf /opt/adios2-src
+ENV ADIOS2_DIR=/opt/adios2
+ENV HDF5_ROOT=/usr
+
+# other packages + python
+RUN apt-get update && \
+    apt-get install -y bc gpg curl ssh vim emacs bat fzf ffmpeg hdf5-tools software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && apt-get install -y python3.12-dev python3.12-venv
+
+# eza
+RUN mkdir -p /etc/apt/keyrings && \
+    wget -qO- https://raw.githubusercontent.com/eza-community/eza/main/deb.asc | gpg --dearmor -o /etc/apt/keyrings/gierens.gpg && \
+    echo "deb [signed-by=/etc/apt/keyrings/gierens.gpg] http://deb.gierens.de stable main" | tee /etc/apt/sources.list.d/gierens.list && \
+    chmod 644 /etc/apt/keyrings/gierens.gpg /etc/apt/sources.list.d/gierens.list && \
+    apt-get update && apt-get install -y eza
+
+# cleanup
+RUN apt-get clean && \
+    apt-get autoclean && \
+    apt-get autoremove -y && \
+    rm -rf /var/lib/cache/* && \
+    rm -rf /var/lib/log/* && \
+    rm -rf /var/lib/apt/lists/*
+
+# python
+RUN ln -s $(which python3.12) /bin/python && \
+    python -m venv /opt/venv && \
+    /opt/venv/bin/pip install --upgrade pip && \
+    /opt/venv/bin/pip install black numpy myplotlib nt2py jupyterlab ipykernel
+ENV VIRTUAL_ENV=/opt/venv
+
+# user environment
+ARG HOME=/root
+WORKDIR $HOME
+RUN echo "alias ls='eza -a --sort=type'" >> $HOME/.bashrc && \
+    echo "alias ll='eza -a --long --header --sort=type --time-style=long-iso'" >> $HOME/.bashrc && \
+    echo "alias cat='batcat -pp'" >> $HOME/.bashrc
+
+RUN curl -sS https://starship.rs/install.sh | sh -s -- --yes && \
+    echo eval "\$(starship init bash)" >> $HOME/.bashrc && \
+    mkdir -p $HOME/.config && \
+    starship preset pure-preset -o $HOME/.config/starship.toml && \
+    sed -i 's/$python\\//g' $HOME/.config/starship.toml
+ENV STARSHIP_CONFIG=$HOME/.config/starship.toml
+
+ENV PATH=/opt/adios2/bin:/opt/venv/bin:$PATH

dev/Dockerfile.cuda

@@ -1,77 +1,15 @@
+# syntax = devthefuture/dockerfile-x
+
 FROM nvidia/cuda:12.2.2-devel-ubuntu22.04
-ENV CUDA_HOME=/usr/local/cuda
 MAINTAINER "@haykh"
 
-ARG DEBIAN_FRONTEND=noninteractive
-ENV DISPLAY=host.docker.internal:0.0
-
-# basic packages
-RUN apt-get update 
-RUN apt-get install -y bc zsh git wget curl ssh build-essential cmake vim emacs bat fzf
-RUN apt-get install -y ffmpeg libhdf5-dev python3 python3-pip python3-venv 
-
-# eza
-RUN apt install -y gpg
-RUN mkdir -p /etc/apt/keyrings
-RUN wget -qO- https://raw.githubusercontent.com/eza-community/eza/main/deb.asc | gpg --dearmor -o /etc/apt/keyrings/gierens.gpg
-RUN echo "deb [signed-by=/etc/apt/keyrings/gierens.gpg] http://deb.gierens.de stable main" | tee /etc/apt/sources.list.d/gierens.list
-RUN chmod 644 /etc/apt/keyrings/gierens.gpg /etc/apt/sources.list.d/gierens.list
-RUN apt update
-RUN apt install -y eza
-
-# cleanup
-RUN apt-get clean && \
-    apt-get autoclean && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/cache/* && \
-    rm -rf /var/lib/log/*
-
-# adios2
-ENV HDF5_ROOT=/usr
-RUN git clone https://github.com/ornladios/ADIOS2.git /opt/adios2-src
-RUN cd /opt/adios2-src && \
-    cmake -B build \
-    -D CMAKE_CXX_STANDARD=17 \
-    -D CMAKE_CXX_EXTENSIONS=OFF \
-    -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-    -D BUILD_SHARED_LIBS=ON \
-    -D ADIOS2_USE_HDF5=ON \
-    -D ADIOS2_USE_Python=OFF \
-    -D ADIOS2_USE_Fortran=OFF \
-    -D ADIOS2_USE_ZeroMQ=OFF \
-    -D BUILD_TESTING=OFF \
-    -D ADIOS2_BUILD_EXAMPLES=OFF \
-    -D ADIOS2_USE_MPI=OFF \
-    -D ADIOS2_HAVE_HDF5_VOL=OFF \
-    -D CMAKE_INSTALL_PREFIX=/opt/adios2 && \
-    cmake --build build && \
-    cmake --install build
-ENV ADIOS2_DIR=/opt/adios2
-
-# python
-RUN ln -s /bin/python3 /bin/python
-RUN python3 -m venv /opt/venv && \
-    /opt/venv/bin/pip install --upgrade pip && \
-    /opt/venv/bin/pip install black numpy myplotlib nt2py jupyterlab ipykernel
-ENV VIRTUAL_ENV=/opt/venv
-
-# user environment
-ENV USER=ntt
-WORKDIR /home
-ENV PATH=/opt/adios2/bin:/usr/local/cuda/bin:/opt/venv/bin:$PATH
-RUN echo "alias ls='eza -a --sort=type'" >> /root/.bashrc
-RUN echo "alias ll='eza -a --long --header --sort=type --time-style=long-iso'" >> /root/.bashrc
-RUN echo "alias cat='batcat -pp'" >> /root/.bashrc
+ENV CUDA_HOME=/usr/local/cuda
+ENV PATH=/usr/local/cuda/bin:$PATH
 
-RUN curl -sS https://starship.rs/install.sh | sh -s -- --yes
-RUN echo eval "\$(starship init bash)" >> /root/.bashrc
-RUN mkdir -p /home/.config
-RUN starship preset pure-preset -o /home/.config/starship.toml
-RUN sed -i 's/$python\\//g' /home/.config/starship.toml
-ENV STARSHIP_CONFIG=/home/.config/starship.toml
+INCLUDE Dockerfile.common
 
 # welcome message
-COPY welcome.cuda /home/.welcome.cuda
-RUN echo '/usr/bin/bash ./.welcome.cuda' > /etc/profile.d/welcome.sh
+COPY welcome.cuda /root/.welcome.cuda
+RUN echo '/usr/bin/bash /root/.welcome.cuda' | tee -a /etc/profile.d/welcome.sh
 
 ENTRYPOINT ["/usr/bin/bash", "-l"]

dev/Dockerfile.rocm

@@ -0,0 +1,26 @@
+# syntax = devthefuture/dockerfile-x
+
+FROM rocm/rocm-terminal:latest
+MAINTAINER "@haykh"
+
+USER root
+ENV PATH=/opt/rocm/bin:$PATH
+
+INCLUDE Dockerfile.common
+
+# additional ROCm packages
+RUN git clone -b release/rocm-rel-6.1.1.1 https://github.com/ROCm/rocThrust.git /opt/rocthrust-src && \
+    git clone -b release/rocm-rel-6.1.00.36 https://github.com/ROCm/rocPRIM.git /opt/rocprim-src && \
+    cd /opt/rocthrust-src && ./install --install && \
+    cd /opt/rocprim-src && ./install --install && \
+    rm -rf /opt/rocthrust-src /opt/rocprim-src
+
+ENV CMAKE_PREFIX_PATH=/opt/rocm 
+ENV CC=hipcc 
+ENV CXX=hipcc
+
+# welcome message
+COPY welcome.rocm /root/.welcome.rocm
+RUN echo '/usr/bin/bash /root/.welcome.rocm' | tee -a /etc/profile.d/welcome.sh
+
+ENTRYPOINT ["/usr/bin/bash", "-l"]

dev/welcome.cuda

@@ -47,6 +47,13 @@ else
   cmake_version=${error_msg}
 fi
 
+hdf5_or_error=$(h5cc -showconfig | grep Version 2>&1)
+if [ $? -eq 0 ]; then
+  hdf5_version=${FgYellow}$(h5cc -showconfig | grep Version | xargs)${Reset}
+else
+  hdf5_version=${error_msg}
+fi
+
 adios2_or_error=$(adios2-config -v 2>&1)
 if [ $? -eq 0 ]; then
   adios2_version=${FgYellow}$(adios2-config -v | grep ADIOS)${Reset}
@@ -76,6 +83,7 @@ ${Reset}${Underscore}Entity CUDA container${Reset}${FgBlue}               \\/__/
   - ${gcc_version}
   - ${nvcc_version}
   - ${cmake_version}
+  - ${hdf5_version}
   - ${adios2_version}
   - ${python_version}
 

dev/welcome.rocm

@@ -0,0 +1,121 @@
+#!/bin/sh
+Reset="\x1b[0m"
+Bright="\x1b[1m"
+Dim="\x1b[2m"
+Underscore="\x1b[4m"
+Blink="\x1b[5m"
+Reverse="\x1b[7m"
+Hidden="\x1b[8m"
+
+FgBlack="\x1b[30m"
+FgRed="\x1b[31m"
+FgGreen="\x1b[32m"
+FgYellow="\x1b[33m"
+FgBlue="\x1b[34m"
+FgMagenta="\x1b[35m"
+FgCyan="\x1b[36m"
+FgWhite="\x1b[37m"
+
+BgBlack="\x1b[40m"
+BgRed="\x1b[41m"
+BgGreen="\x1b[42m"
+BgYellow="\x1b[43m"
+BgBlue="\x1b[44m"
+BgMagenta="\x1b[45m"
+BgCyan="\x1b[46m"
+BgWhite="\x1b[47m"
+
+error_msg=${FgRed}"NOT FOUND"${Reset}
+gcc_or_error=$(gcc --version 2>&1)
+if [ $? -eq 0 ]; then
+  gcc_version=${FgYellow}$(gcc --version | grep gcc)${Reset}
+else
+  gcc_version=${error_msg}
+fi
+
+rocm_or_error=$(hipcc --version | grep HIP 2>&1)
+if [ $? -eq 0 ]; then
+  rocm_version=${FgYellow}$(hipcc --version | grep HIP)${Reset}
+else
+  rocm_version=${error_msg}
+fi
+
+cmake_or_error=$(cmake --version 2>&1)
+if [ $? -eq 0 ]; then
+  cmake_version=${FgYellow}$(cmake --version | grep ver)${Reset}
+else
+  cmake_version=${error_msg}
+fi
+
+hdf5_or_error=$(h5cc -showconfig | grep Version 2>&1)
+if [ $? -eq 0 ]; then
+  hdf5_version=${FgYellow}$(h5cc -showconfig | grep Version | xargs)${Reset}
+else
+  hdf5_version=${error_msg}
+fi
+
+adios2_or_error=$(adios2-config -v 2>&1)
+if [ $? -eq 0 ]; then
+  adios2_version=${FgYellow}$(adios2-config -v | grep ADIOS)${Reset}
+else
+  adios2_version=${error_msg}
+fi
+
+python_or_error=$(python --version 2>&1)
+if [ $? -eq 0 ]; then
+  python_version=${FgYellow}$(python --version)${Reset}
+else
+  python_version=${error_msg}
+fi
+
+str=$(cat <<EOF
+${FgBlue}              __        __ 
+             /\\ \\__  __/\\ \\__ 
+    __    ___\\ \\  _\\/\\_\\ \\  _\\  __  __ 
+  / __ \\ / __ \\ \\ \\/\\/\\ \\ \\ \\/ /\\ \\/\\ \\ 
+ /\\  __//\\ \\/\\ \\ \\ \\_\\ \\ \\ \\ \\_\\ \\ \\_\\ \\  __ 
+ \\ \\____\\ \\_\\ \\_\\ \\__\\\\\\ \\_\\ \\__\\\\\\ \\____ \\/\\_\\ 
+  \\/____/\\/_/\\/_/\\/__/ \\/_/\\/__/ \\/___/  \\/_/ 
+                                    /\\___/ 
+${Reset}${Underscore}Entity ROCm container${Reset}${FgBlue}               \\/__/${Reset} 
+
+${FgRed}This image is experimental, and may not work out of the box.${Reset}
+
+## Installed packages:
+  - ${gcc_version}
+  - ${rocm_version}
+  - ${cmake_version}
+  - ${hdf5_version}
+  - ${adios2_version}
+  - ${python_version}
+
+## Usage
+
+* Clone the code:
+  ${FgGreen}$ git clone https://github.com/entity-toolkit/entity.git && cd entity${Reset}
+
+* Configure the build for your AMD GPU architecture:
+  ${FgGreen}$ cmake -B build -D pgen=... -D output=ON -D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_***=ON${Reset}
+  All available architectures can be found here: https://kokkos.org/kokkos-core-wiki/keywords.html#nvidia-gpus.
+
+* Build the code:
+  ${FgGreen}$ cmake --build build -j \$(nproc)${Reset}
+
+* The executable will be generated as \`build/src/entity.xc\`. Run it by specifying the input file:
+  ${FgGreen}$ build/src/entity.xc -input <input_file>${Reset}
+
+## Testing the code
+
+* To build the tests, remove the \`build\` directory, and preconfigure with:
+  ${FgGreen}$ cmake -B build -D TESTS=ON -D output=ON -D Kokkos_ENABLE_HIP=ON -D Kokkos_ARCH_***=ON${Reset}
+
+* Then run the tests with:
+  ${FgGreen}$ ctest --test-dir build${Reset}
+
+
+${Red}Enjoy!${Reset} 🚀
+EOF
+)
+
+
+printf "$str\n"

docker-compose.yml

@@ -1,12 +1,37 @@
+x-common-config: &common-config
+  stdin_open: true
+  tty: true
+  ports:
+    - "8080:8080"
+  volumes:
+    - type: bind
+      source: .
+      target: /root/entity/
+
+x-cuda-base: &cuda-base
+  image: morninbru/entity:cuda
+
+x-rocm-base: &rocm-base
+  image: morninbru/entity:rocm
+
 services:
+  entity-cuda-compilers:
+    container_name: entity_cuda_compilers
+    <<: [*cuda-base, *common-config]
   entity-cuda:
-    image: morninbru/entity:cuda
     container_name: entity_cuda
-    stdin_open: true
-    tty: true
-    ports:
-      - "8080:8080"
-    volumes:
-      - type: bind
-        source: .
-        target: /home/entity/
+    runtime: nvidia
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+    <<: [*cuda-base, *common-config]
+  entity-rocm-compilers:
+    container_name: entity_rocm_compilers
+    <<: [*common-config, *rocm-base]
+  entity-rocm:
+    container_name: entity_rocm
+    devices:
+      - "/dev/fdk"
+      - "/dev/dri"
+    security_opt:
+      - seccomp:unconfined
+    <<: [*common-config, *rocm-base]