Use latest client package in Dockerfiles and automatically build+publish client container

.github/workflows/build-publish-containers.yml

@@ -0,0 +1,115 @@
+name: Build and publish EESSI container images
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - containers/Dockerfile.EESSI-*
+      - containers/build-or-download-cvmfs-*.sh
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - containers/Dockerfile.EESSI-client*
+      - containers/build-or-download-cvmfs-rpm.sh
+
+  # also rebuild the containers for new releases of filesystem-layer:
+  # this often means that we have a new CVMFS configuration package
+  release:
+    types: [published]
+
+jobs:
+  build_and_publish:
+    name: Build and publish image
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      matrix:
+        tag: ['client-pilot:centos7', 'build-node:debian10']
+        platform: [amd64, arm64, ppc64le]
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ matrix.platform }}-${{ matrix.tag }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ matrix.platform }}-${{ matrix.tag }}
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1 
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Convert and store repository owner in lowercase, replace colon in tag names by hyphen
+        run: |
+          echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
+          tag=${{ matrix.tag }}
+          echo DOCKERFILE=Dockerfile.EESSI-${tag//:/-} >> $GITHUB_ENV
+
+      - name: Build and push to GitHub Packages
+        uses: docker/build-push-action@v2
+        with:
+          tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-${{ matrix.platform }}
+          file: containers/${{ env.DOCKERFILE }}
+          platforms: linux/${{ matrix.platform }}
+          push: ${{ github.event_name != 'pull_request' }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  create_manifest:
+    name: Create the Docker manifest
+    needs: build_and_publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      matrix:
+        tag: ['client-pilot:centos7', 'build-node:debian10']
+    steps:
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Convert and store repository owner in lowercase
+        run: |
+          echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
+
+      - name: Run docker manifest
+        if: github.event_name != 'pull_request'
+        run: |
+          docker manifest create \
+            ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \
+            --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \
+            --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \
+            --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-ppc64le
+
+      - name: Push manifest
+        if: github.event_name != 'pull_request'
+        run: docker manifest push ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}

.github/workflows/test-containers.yml

@@ -8,4 +8,4 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Build the Docker image
-        run: docker build . --file containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
+        run: docker build . --file containers/Dockerfile.EESSI-client-pilot-centos7

containers/Dockerfile.EESSI-build-node-debian10

@@ -0,0 +1,27 @@
+ARG cvmfsversion=2.8.1
+ARG fuseoverlayfsversion=0.4.1
+
+FROM debian:10.6 AS prepare-deb
+ARG cvmfsversion
+COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh
+RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
+
+
+FROM debian:10.6
+ARG cvmfsversion
+
+COPY --from=prepare-deb /root/deb /root/deb
+
+RUN apt-get update
+RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev
+RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
+            /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
+            /root/deb/cvmfs-config-default_latest_all.deb \
+            /root/deb/cvmfs-config-eessi_latest_all.deb
+
+RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
+
+RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
+
+RUN useradd -ms /bin/bash eessi

containers/Dockerfile.EESSI-client-pilot-centos7

@@ -0,0 +1,36 @@
+ARG cvmfsversion=2.8.1
+ARG fuseoverlayfsversion=0.4.1
+
+FROM centos:7 AS prepare-rpm
+ARG cvmfsversion
+COPY ./containers/build-or-download-cvmfs-rpms.sh /build-or-download-cvmfs-rpms.sh
+RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
+
+
+FROM centos:7 AS build-fuse-overlayfs
+ARG fuseoverlayfsversion
+RUN yum install -y wget fuse3-devel autoconf automake gcc make tar
+RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
+  && tar xzf v${fuseoverlayfsversion}.tar.gz \
+  && cd fuse-overlayfs-${fuseoverlayfsversion} \
+  && ./autogen.sh && ./configure && make && make install
+
+
+FROM centos:7
+ARG cvmfsversion
+
+COPY --from=prepare-rpm /root/rpmbuild/RPMS /root/rpmbuild/RPMS
+COPY --from=build-fuse-overlayfs /usr/local/bin/fuse-overlayfs /usr/local/bin/fuse-overlayfs
+
+RUN yum install -y sudo vim openssh-clients
+RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$(uname -m).rpm \
+                   /root/rpmbuild/RPMS/$(uname -m)/cvmfs-fuse3-${cvmfsversion}-1.el7.$(uname -m).rpm \
+                   http://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default-latest.noarch.rpm
+RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
+
+RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
+
+RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
+
+RUN useradd -ms /bin/bash eessi

containers/README.md

@@ -5,25 +5,41 @@ This directory contains recipes for containers that are useful in the scope of t
 ## Client container
 
 Container to provide easy access to EESSI pilot repository,
-see https://hub.docker.com/repository/docker/eessi/client-pilot and https://eessi.github.io/docs/pilot.
-
-### Build container + push to Docker Hub
-
-Note: the `docker push` part of the script assumes your Docker Hub creditionals are known
-(can be done via `docker login docker.io`, for example).
-
-```shell
-./docker_build_push.sh
-```
-This will build the container for the architecture of your host (e.g. `x86_64` or `aarch64`), and push the image to Docker Hub.
+see https://github.com/users/EESSI/packages/container/package/client-pilot and https://eessi.github.io/docs/pilot.
+This container image is based on CentOS 7, and gets automatically built and pushed to the GitHub Container Registry when one of its
+source files (the Dockerfile or the script that generates the CernVM-FS RPMs) gets changed,
+or when a new version of the filesystem-layer repository is released.
 
 ### Run (using Singularity)
 
 ```
 mkdir -p /tmp/$USER/{var-lib-cvmfs,var-run-cvmfs,home}
 export SINGULARITY_BIND="/tmp/$USER/var-run-cvmfs:/var/run/cvmfs,/tmp/$USER/var-lib-cvmfs:/var/lib/cvmfs"
 export SINGULARITY_HOME="/tmp/$USER/home:/home/$USER"
-export EESSI_CONFIG="container:cvmfs2 cvmfs-config.eessi-hpc.org /cvmfs/cvmfs-config.eessi-hpc.org"
 export EESSI_PILOT="container:cvmfs2 pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org"
-singularity shell --fusemount "$EESSI_CONFIG" --fusemount "$EESSI_PILOT" docker://eessi/client-pilot:centos7-$(uname -m)
+singularity shell --fusemount "$EESSI_PILOT" docker://ghcr.io/EESSI/client-pilot:centos7
 ```
+
+## Build node container
+
+Container that can be used to build and install software to /cvmfs by leveraging `fuse-overlayfs` for
+providing a writable overlay.
+The container image is based on Debian 10.6, and gets automatically built and pushed to the GitHub Container Registry when one of its
+source files (the Dockerfile or the script that generates the CernVM-FS deb packages) gets changed,
+or when a new version of the filesystem-layer repository is released.
+
+### Run (using Singularity)
+```
+export EESSI_TMPDIR=/tmp/$USER/EESSI
+mkdir -p $EESSI_TMPDIR
+mkdir -p $EESSI_TMPDIR/{home,overlay-upper,overlay-work}
+mkdir -p $EESSI_TMPDIR/{var-lib-cvmfs,var-run-cvmfs}
+export SINGULARITY_CACHEDIR=$EESSI_TMPDIR/singularity_cache
+export SINGULARITY_BIND="$EESSI_TMPDIR/var-run-cvmfs:/var/run/cvmfs,$EESSI_TMPDIR/var-lib-cvmfs:/var/lib/cvmfs"
+export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER"
+export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org"
+export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org"
+singularity shell --fusemount "$EESSI_PILOT_READONLY" --fusemount "$EESSI_PILOT_WRITABLE_OVERLAY" docker://eessi/fuse-overlay:debian10-$(uname -m)
+```
+
+For more details about building software, see: https://eessi.github.io/docs/software_layer/build_nodes/

containers/build-or-download-cvmfs-debs.sh

@@ -0,0 +1,25 @@
+cvmfsversion=$1
+arch=$(dpkg --print-architecture)
+
+apt-get update
+apt-get install -y wget
+if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
+then
+    apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
+    cd /tmp
+    wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
+    tar xzf cvmfs-${cvmfsversion}.tar.gz
+    cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs
+    mkdir /root/deb
+    sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 ppc64el/' ../../packaging/debian/cvmfs/control.in
+    ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb
+else
+    mkdir -p /root/deb
+    cd /root/deb
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs_${cvmfsversion}~1+debian10_${arch}.deb
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-fuse3_${cvmfsversion}~1+debian10_${arch}.deb
+fi
+
+cd /root/deb
+wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default_latest_all.deb
+wget https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb

containers/build-or-download-cvmfs-rpms.sh

@@ -0,0 +1,15 @@
+cvmfsversion=$1
+arch=$(uname -m)
+
+yum install -y wget
+if [ "$arch" = "ppc64le" ]
+then
+    yum install -y epel-release
+    yum install -y rpm-build checkpolicy cmake fuse-devel fuse3-devel gcc gcc-c++ golang libattr-devel libcap-devel libuuid-devel openssl-devel python2-devel python2-setuptools selinux-policy-devel valgrind-devel hardlink selinux-policy-targeted
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.src.rpm && rpmbuild --rebuild cvmfs-${cvmfsversion}-1.el7.src.rpm
+else
+    mkdir -p /root/rpmbuild/RPMS/${arch}
+    cd /root/rpmbuild/RPMS/${arch}
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.${arch}.rpm
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-fuse3-${cvmfsversion}-1.el7.${arch}.rpm
+fi