From 4758da5c9d873d001a3de2683e086d5e20512521 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 22 Apr 2021 15:01:31 +0200
Subject: [PATCH 01/56] use new/latest config packages

---
 containers/Dockerfile.EESSI-client-pilot-centos7-aarch64 | 4 ++--
 containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le | 4 ++--
 containers/Dockerfile.EESSI-client-pilot-centos7-x86_64  | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7-aarch64 b/containers/Dockerfile.EESSI-client-pilot-centos7-aarch64
index 23ca37d4..a71e652d 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7-aarch64
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7-aarch64
@@ -3,10 +3,10 @@ FROM docker.io/arm64v8/centos:7
 RUN yum install -y http://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs-release-latest.noarch.rpm \
  && yum install -y cvmfs cvmfs-fuse3 --nogpgcheck \
  && yum install -y cvmfs-config-default sudo vim openssh-clients \
- && yum install -y https://github.com/EESSI/filesystem-layer/releases/download/v0.2.3/cvmfs-config-eessi-0.2.3-1.noarch.rpm
+ && yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
 
 RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_HTTP_PROXY="DIRECT"' >> /etc/cvmfs/default.local
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
 
 RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
 
diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le b/containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le
index 91ca03d2..3d402a30 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le
@@ -1,6 +1,6 @@
 FROM docker.io/ppc64le/centos:7
 ARG cvmfsversion=2.8.0
-ARG cvmfsconfig=https://github.com/EESSI/filesystem-layer/releases/download/v0.2.3/cvmfs-config-eessi-0.2.3-1.noarch.rpm
+ARG cvmfsconfig=https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
 
 RUN yum install -y sudo vim openssh-clients cmake wget make unzip patch valgrind bzip2
 
@@ -24,7 +24,7 @@ RUN yum install -y ${cvmfsconfig}
 RUN yum remove -y fuse && yum install -y fuse3
 
 RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_HTTP_PROXY="DIRECT"' >> /etc/cvmfs/default.local
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
 
 RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
 
diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7-x86_64 b/containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
index 3859415e..b18281b9 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
@@ -2,10 +2,10 @@ FROM docker.io/library/centos:7
 
 RUN yum install -y http://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs-release-latest.noarch.rpm \
  && yum install -y cvmfs cvmfs-config-default cvmfs-fuse3 sudo vim openssh-clients \
- && yum install -y https://github.com/EESSI/filesystem-layer/releases/download/v0.2.3/cvmfs-config-eessi-0.2.3-1.noarch.rpm
+ && yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
 
 RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_HTTP_PROXY="DIRECT"' >> /etc/cvmfs/default.local
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
 
 RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
 

From 2a8639f7f19855e820fcd4958f83d85342d72f24 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 22 Apr 2021 15:32:54 +0200
Subject: [PATCH 02/56] don't mount cvmfs-config repo in container

---
 containers/README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/containers/README.md b/containers/README.md
index b1e1336d..73a044da 100644
--- a/containers/README.md
+++ b/containers/README.md
@@ -23,7 +23,6 @@ This will build the container for the architecture of your host (e.g. `x86_64` o
 mkdir -p /tmp/$USER/{var-lib-cvmfs,var-run-cvmfs,home}
 export SINGULARITY_BIND="/tmp/$USER/var-run-cvmfs:/var/run/cvmfs,/tmp/$USER/var-lib-cvmfs:/var/lib/cvmfs"
 export SINGULARITY_HOME="/tmp/$USER/home:/home/$USER"
-export EESSI_CONFIG="container:cvmfs2 cvmfs-config.eessi-hpc.org /cvmfs/cvmfs-config.eessi-hpc.org"
 export EESSI_PILOT="container:cvmfs2 pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org"
-singularity shell --fusemount "$EESSI_CONFIG" --fusemount "$EESSI_PILOT" docker://eessi/client-pilot:centos7-$(uname -m)
+singularity shell --fusemount "$EESSI_PILOT" docker://eessi/client-pilot:centos7-$(uname -m)
 ```

From 2099b5235b50d947862f1738f048ef02b1973f52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:03:12 +0200
Subject: [PATCH 03/56] add multistage and multiarch Dockerfile

---
 .../workflows/build-publish-containers.yml    | 51 +++++++++++++++++++
 .../Dockerfile.EESSI-client-pilot-centos7     | 26 ++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 .github/workflows/build-publish-containers.yml
 create mode 100644 containers/Dockerfile.EESSI-client-pilot-centos7

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
new file mode 100644
index 00000000..99ec7583
--- /dev/null
+++ b/.github/workflows/build-publish-containers.yml
@@ -0,0 +1,51 @@
+name: Build and publish EESSI client Docker image
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - containers/Dockerfile.EESSI-client*
+      - containers/build-or-download-cvmfs-rpm.sh
+      - .github/workflows/build-publish-containers.yml
+
+  # also rebuild the containers for new releases of filesystem-layer:
+  # this often means that we have a new CVMFS configuration package
+  release:
+    types: [published]
+ 
+jobs:
+  build_and_publish:
+    name: Build and publish image
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      matrix:
+        os: [linux]
+        arch: [amd64, arm64, ppc64le]
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1 
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push to GitHub Packages
+        uses: docker/build-push-action@v2
+        with:
+          tags: ghcr.io/EESSI/client-pilot:centos7
+          file: containers/Dockerfile.EESSI-client-pilot-centos7
+          platforms: ${{ matrix.os }}/${{ matrix.arch }}
+          push: true
diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
new file mode 100644
index 00000000..0ddc16ce
--- /dev/null
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -0,0 +1,26 @@
+ARG cvmfsversion=2.8.1
+
+
+FROM centos:7 AS prepare-rpm
+ARG cvmfsversion
+COPY ./containers/build-or-download-cvmfs-rpm.sh /build-or-download-cvmfs-rpm.sh
+RUN sh /build-or-download-cvmfs-rpm.sh ${cvmfsversion}
+
+
+FROM centos:7
+ARG cvmfsversion
+
+COPY --from=prepare-rpm /root/rpmbuild/RPMS /root/rpmbuild/RPMS
+
+RUN yum install -y sudo vim openssh-clients
+RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$(uname -m).rpm \
+                   /root/rpmbuild/RPMS/$(uname -m)/cvmfs-fuse3-${cvmfsversion}-1.el7.$(uname -m).rpm \
+                   http://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default-latest.noarch.rpm
+RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
+
+RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
+  && echo 'CVMFS_HTTP_PROXY="DIRECT"' >> /etc/cvmfs/default.local
+
+RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
+
+RUN useradd -ms /bin/bash eessi

From d3e90fc65d7f6f0c2cbb3d400e3e51ce80f7ca1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:04:32 +0200
Subject: [PATCH 04/56] script for getting/making CVMFS rpms

---
 containers/build-or-download-cvmfs-rpms.sh | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 containers/build-or-download-cvmfs-rpms.sh

diff --git a/containers/build-or-download-cvmfs-rpms.sh b/containers/build-or-download-cvmfs-rpms.sh
new file mode 100644
index 00000000..fe256427
--- /dev/null
+++ b/containers/build-or-download-cvmfs-rpms.sh
@@ -0,0 +1,15 @@
+cvmfsversion=$1
+arch=$(uname -m)
+
+yum install -y wget
+if [ "$arch" = "ppc64le" ]
+then
+    yum install -y epel-release
+    yum install -y rpm-build checkpolicy cmake fuse-devel fuse3-devel gcc gcc-c++ golang libattr-devel libcap-devel libuuid-devel openssl-devel python2-devel python2-setuptools selinux-policy-devel valgrind-devel hardlink selinux-policy-targeted
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.src.rpm && rpmbuild --rebuild cvmfs-${cvmfsversion}-1.el7.src.rpm
+else
+    mkdir -p /root/rpmbuild/RPMS/${arch}
+    cd /root/rpmbuild/RPMS/${arch}
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.${arch}.rpm
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-fuse3-${cvmfsversion}-1.el7.${arch}.rpm
+fi

From 28569c86d8e60dc8326530815a511aa1d2a7b05d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:05:16 +0200
Subject: [PATCH 05/56] remove old arch-specific dockerfiles and push script

---
 .../Dockerfile.EESSI-client-pilot-centos7     |  4 +--
 ...kerfile.EESSI-client-pilot-centos7-aarch64 | 13 --------
 ...kerfile.EESSI-client-pilot-centos7-ppc64le | 31 -------------------
 ...ckerfile.EESSI-client-pilot-centos7-x86_64 | 12 -------
 containers/docker_build_push.sh               |  9 ------
 5 files changed, 2 insertions(+), 67 deletions(-)
 delete mode 100644 containers/Dockerfile.EESSI-client-pilot-centos7-aarch64
 delete mode 100644 containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le
 delete mode 100644 containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
 delete mode 100755 containers/docker_build_push.sh

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
index 0ddc16ce..56d49e89 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -3,8 +3,8 @@ ARG cvmfsversion=2.8.1
 
 FROM centos:7 AS prepare-rpm
 ARG cvmfsversion
-COPY ./containers/build-or-download-cvmfs-rpm.sh /build-or-download-cvmfs-rpm.sh
-RUN sh /build-or-download-cvmfs-rpm.sh ${cvmfsversion}
+COPY ./containers/build-or-download-cvmfs-rpms.sh /build-or-download-cvmfs-rpms.sh
+RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
 
 
 FROM centos:7
diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7-aarch64 b/containers/Dockerfile.EESSI-client-pilot-centos7-aarch64
deleted file mode 100644
index a71e652d..00000000
--- a/containers/Dockerfile.EESSI-client-pilot-centos7-aarch64
+++ /dev/null
@@ -1,13 +0,0 @@
-FROM docker.io/arm64v8/centos:7
-
-RUN yum install -y http://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs-release-latest.noarch.rpm \
- && yum install -y cvmfs cvmfs-fuse3 --nogpgcheck \
- && yum install -y cvmfs-config-default sudo vim openssh-clients \
- && yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
-
-RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
-
-RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
-
-RUN useradd -ms /bin/bash eessi
diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le b/containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le
deleted file mode 100644
index 3d402a30..00000000
--- a/containers/Dockerfile.EESSI-client-pilot-centos7-ppc64le
+++ /dev/null
@@ -1,31 +0,0 @@
-FROM docker.io/ppc64le/centos:7
-ARG cvmfsversion=2.8.0
-ARG cvmfsconfig=https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
-
-RUN yum install -y sudo vim openssh-clients cmake wget make unzip patch valgrind bzip2
-
-RUN yum install -y gcc gcc-c++ sqlite-devel python-devel libcap-devel libuuid-devel attr \
- && yum install -y fuse-devel fuse3-devel zlib-devel openssl-devel
-
-# build CernVM-FS from source (no aarch64 Debian packages available)
-RUN  wget https://github.com/cvmfs/cvmfs/archive/cvmfs-${cvmfsversion}.tar.gz && \
-  tar xfz cvmfs-${cvmfsversion}.tar.gz && \
-  cd cvmfs*${cvmfsversion}/ && \
-  mkdir build && \
-  cd build && \
-  cmake .. -DBUILD_SERVER=no -DBUILD_SERVER_DEBUG=no -DDBUILD_SHRINKWRAP=no && \
-  make -j $(nproc) && \
-  sudo make install && \
-  cd / && \
-  rm -r cvmfs*${cvmfsversion}*
-
-RUN yum install -y ${cvmfsconfig}
-
-RUN yum remove -y fuse && yum install -y fuse3
-
-RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
-
-RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
-
-RUN useradd -ms /bin/bash eessi
diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7-x86_64 b/containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
deleted file mode 100644
index b18281b9..00000000
--- a/containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM docker.io/library/centos:7
-
-RUN yum install -y http://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs-release-latest.noarch.rpm \
- && yum install -y cvmfs cvmfs-config-default cvmfs-fuse3 sudo vim openssh-clients \
- && yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
-
-RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
-
-RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
-
-RUN useradd -ms /bin/bash eessi
diff --git a/containers/docker_build_push.sh b/containers/docker_build_push.sh
deleted file mode 100755
index 045108ae..00000000
--- a/containers/docker_build_push.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-os="centos7"
-cpu_arch=$(uname -m)
-tag="eessi/client-pilot:${os}-${cpu_arch}"
-
-docker build --no-cache -f Dockerfile.EESSI-client-pilot-${os}-${cpu_arch} -t ${tag} .
-
-docker push ${tag}

From 3469c8aadcddac5e8a6ef6d797c834a28739996c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:08:57 +0200
Subject: [PATCH 06/56] update README for new container image

---
 containers/README.md | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/containers/README.md b/containers/README.md
index 73a044da..faa898eb 100644
--- a/containers/README.md
+++ b/containers/README.md
@@ -5,17 +5,10 @@ This directory contains recipes for containers that are useful in the scope of t
 ## Client container
 
 Container to provide easy access to EESSI pilot repository,
-see https://hub.docker.com/repository/docker/eessi/client-pilot and https://eessi.github.io/docs/pilot.
-
-### Build container + push to Docker Hub
-
-Note: the `docker push` part of the script assumes your Docker Hub creditionals are known
-(can be done via `docker login docker.io`, for example).
-
-```shell
-./docker_build_push.sh
-```
-This will build the container for the architecture of your host (e.g. `x86_64` or `aarch64`), and push the image to Docker Hub.
+see https://github.com/users/EESSI/packages/container/package/client-pilot and https://eessi.github.io/docs/pilot.
+This container image gets automatically built and pushed to the GitHub Container Registry when one of its
+source files (the Dockerfile or the script that generates the CernVM-FS RPMs) gets changed,
+or when a new version of the filesystem-layer repository is released.
 
 ### Run (using Singularity)
 
@@ -24,5 +17,5 @@ mkdir -p /tmp/$USER/{var-lib-cvmfs,var-run-cvmfs,home}
 export SINGULARITY_BIND="/tmp/$USER/var-run-cvmfs:/var/run/cvmfs,/tmp/$USER/var-lib-cvmfs:/var/lib/cvmfs"
 export SINGULARITY_HOME="/tmp/$USER/home:/home/$USER"
 export EESSI_PILOT="container:cvmfs2 pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org"
-singularity shell --fusemount "$EESSI_PILOT" docker://eessi/client-pilot:centos7-$(uname -m)
+singularity shell --fusemount "$EESSI_PILOT" docker://ghcr.io/EESSI/client-pilot:centos7
 ```

From 012ac18f3e97ca33bd2dc5251a89c2ddebadd54e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:11:12 +0200
Subject: [PATCH 07/56] modify test-containers workflow for new dockerfile

---
 .github/workflows/test-containers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test-containers.yml b/.github/workflows/test-containers.yml
index 51d6c3f1..dab7519c 100644
--- a/.github/workflows/test-containers.yml
+++ b/.github/workflows/test-containers.yml
@@ -8,4 +8,4 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Build the Docker image
-        run: docker build . --file containers/Dockerfile.EESSI-client-pilot-centos7-x86_64
+        run: docker build . --file containers/Dockerfile.EESSI-client-pilot-centos7

From 3aba5adc75db3effc397acc563a5e6946d66e40b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:29:42 +0200
Subject: [PATCH 08/56] don't use strategy matrix, build all in one go

---
 .github/workflows/build-publish-containers.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 99ec7583..addd10cd 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -20,11 +20,6 @@ jobs:
     permissions:
       packages: write
       contents: read
-    strategy:
-      matrix:
-        os: [linux]
-        arch: [amd64, arm64, ppc64le]
-
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
@@ -47,5 +42,5 @@ jobs:
         with:
           tags: ghcr.io/EESSI/client-pilot:centos7
           file: containers/Dockerfile.EESSI-client-pilot-centos7
-          platforms: ${{ matrix.os }}/${{ matrix.arch }}
+          platforms: linux/amd64, linux/arm64, linux/ppc64le
           push: true

From 1e9a8bbb499f223fc2cc4724fdc2037386c31c55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:31:28 +0200
Subject: [PATCH 09/56] also run on PRs, but without pushing image

---
 .github/workflows/build-publish-containers.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index addd10cd..24cfbfb0 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -8,6 +8,10 @@ on:
       - containers/build-or-download-cvmfs-rpm.sh
       - .github/workflows/build-publish-containers.yml
 
+  pull_request:
+    branches:
+      - main
+
   # also rebuild the containers for new releases of filesystem-layer:
   # this often means that we have a new CVMFS configuration package
   release:
@@ -31,6 +35,7 @@ jobs:
         uses: docker/setup-buildx-action@v1
 
       - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v1 
         with:
           registry: ghcr.io
@@ -43,4 +48,4 @@ jobs:
           tags: ghcr.io/EESSI/client-pilot:centos7
           file: containers/Dockerfile.EESSI-client-pilot-centos7
           platforms: linux/amd64, linux/arm64, linux/ppc64le
-          push: true
+          push: ${{ github.event_name != 'pull_request' }}

From 6df928bed1b4e4b676368bccb726d6532792f36c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:34:05 +0200
Subject: [PATCH 10/56] EESSI in tag should be lowercase

---
 .github/workflows/build-publish-containers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 24cfbfb0..f3ba0596 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -45,7 +45,7 @@ jobs:
       - name: Build and push to GitHub Packages
         uses: docker/build-push-action@v2
         with:
-          tags: ghcr.io/EESSI/client-pilot:centos7
+          tags: ghcr.io/eessi/client-pilot:centos7
           file: containers/Dockerfile.EESSI-client-pilot-centos7
           platforms: linux/amd64, linux/arm64, linux/ppc64le
           push: ${{ github.event_name != 'pull_request' }}

From 628675f54ff5f2a68f3cd2f69f8cd0af99997f32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 09:37:43 +0200
Subject: [PATCH 11/56] only run workflow for PRs that change relevant files

---
 .github/workflows/build-publish-containers.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index f3ba0596..7f6402b2 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -11,6 +11,10 @@ on:
   pull_request:
     branches:
       - main
+    paths:
+      - containers/Dockerfile.EESSI-client*
+      - containers/build-or-download-cvmfs-rpm.sh
+      - .github/workflows/build-publish-containers.yml
 
   # also rebuild the containers for new releases of filesystem-layer:
   # this often means that we have a new CVMFS configuration package

From 0cf8db7db492daf0af1204e20519adc40dec3838 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 26 Apr 2021 17:32:51 +0200
Subject: [PATCH 12/56] dont make /cvmfs/cvmfs-config.eessi-hpc.org

---
 containers/Dockerfile.EESSI-client-pilot-centos7 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
index 56d49e89..443c5b90 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -21,6 +21,6 @@ RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/l
 RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
   && echo 'CVMFS_HTTP_PROXY="DIRECT"' >> /etc/cvmfs/default.local
 
-RUN mkdir -p /cvmfs/{cvmfs-config.eessi-hpc.org,pilot.eessi-hpc.org}
+RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
 
 RUN useradd -ms /bin/bash eessi

From b6e8540469413b820b5f2cb1dc3a490fc7428e29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 28 Apr 2021 13:08:20 +0200
Subject: [PATCH 13/56] don't run on changes in workflow, since that doesn't
 work

---
 .github/workflows/build-publish-containers.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 7f6402b2..9e5bbff6 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -6,7 +6,6 @@ on:
     paths:
       - containers/Dockerfile.EESSI-client*
       - containers/build-or-download-cvmfs-rpm.sh
-      - .github/workflows/build-publish-containers.yml
 
   pull_request:
     branches:
@@ -14,7 +13,6 @@ on:
     paths:
       - containers/Dockerfile.EESSI-client*
       - containers/build-or-download-cvmfs-rpm.sh
-      - .github/workflows/build-publish-containers.yml
 
   # also rebuild the containers for new releases of filesystem-layer:
   # this often means that we have a new CVMFS configuration package

From 6faaf2bc3a9fe01eef5fec40932a93878d92339a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 28 Apr 2021 13:09:25 +0200
Subject: [PATCH 14/56] use variable for repo owner

---
 .github/workflows/build-publish-containers.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 9e5bbff6..ce47334c 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -44,10 +44,14 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Convert and store repository owner in lowercase
+        run: |
+          echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
+
       - name: Build and push to GitHub Packages
         uses: docker/build-push-action@v2
         with:
-          tags: ghcr.io/eessi/client-pilot:centos7
+          tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/client-pilot:centos7
           file: containers/Dockerfile.EESSI-client-pilot-centos7
           platforms: linux/amd64, linux/arm64, linux/ppc64le
           push: ${{ github.event_name != 'pull_request' }}

From c69dc1b63c2bd29ea8da33f37cd47b788a01528d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 28 Apr 2021 14:47:36 +0200
Subject: [PATCH 15/56] cache docker layers

---
 .github/workflows/build-publish-containers.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index ce47334c..05d3b3f0 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -36,6 +36,14 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v1 
@@ -55,3 +63,5 @@ jobs:
           file: containers/Dockerfile.EESSI-client-pilot-centos7
           platforms: linux/amd64, linux/arm64, linux/ppc64le
           push: ${{ github.event_name != 'pull_request' }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new

From 7d03bbdfd2394a8c23fa823afa2b38ce805151ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 29 Apr 2021 09:49:05 +0200
Subject: [PATCH 16/56] add cache rotation step

---
 .github/workflows/build-publish-containers.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 05d3b3f0..55d71717 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -65,3 +65,8 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

From f0b68135671df1950f2823858557f5e3452e5e02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 29 Apr 2021 13:02:45 +0200
Subject: [PATCH 17/56] add comments, trigger rebuild to test cache

---
 containers/build-or-download-cvmfs-rpms.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/containers/build-or-download-cvmfs-rpms.sh b/containers/build-or-download-cvmfs-rpms.sh
index fe256427..c44bb7db 100644
--- a/containers/build-or-download-cvmfs-rpms.sh
+++ b/containers/build-or-download-cvmfs-rpms.sh
@@ -4,10 +4,12 @@ arch=$(uname -m)
 yum install -y wget
 if [ "$arch" = "ppc64le" ]
 then
+    # build RPMs using the source RPMs
     yum install -y epel-release
     yum install -y rpm-build checkpolicy cmake fuse-devel fuse3-devel gcc gcc-c++ golang libattr-devel libcap-devel libuuid-devel openssl-devel python2-devel python2-setuptools selinux-policy-devel valgrind-devel hardlink selinux-policy-targeted
     wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.src.rpm && rpmbuild --rebuild cvmfs-${cvmfsversion}-1.el7.src.rpm
 else
+    # just download the available RPMs
     mkdir -p /root/rpmbuild/RPMS/${arch}
     cd /root/rpmbuild/RPMS/${arch}
     wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.${arch}.rpm

From 91c20b12c2df8a91206b84a296dc9fb3795e53d0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 29 Apr 2021 13:30:50 +0200
Subject: [PATCH 18/56] revert change, test cache

---
 containers/build-or-download-cvmfs-rpms.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/containers/build-or-download-cvmfs-rpms.sh b/containers/build-or-download-cvmfs-rpms.sh
index c44bb7db..fe256427 100644
--- a/containers/build-or-download-cvmfs-rpms.sh
+++ b/containers/build-or-download-cvmfs-rpms.sh
@@ -4,12 +4,10 @@ arch=$(uname -m)
 yum install -y wget
 if [ "$arch" = "ppc64le" ]
 then
-    # build RPMs using the source RPMs
     yum install -y epel-release
     yum install -y rpm-build checkpolicy cmake fuse-devel fuse3-devel gcc gcc-c++ golang libattr-devel libcap-devel libuuid-devel openssl-devel python2-devel python2-setuptools selinux-policy-devel valgrind-devel hardlink selinux-policy-targeted
     wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.src.rpm && rpmbuild --rebuild cvmfs-${cvmfsversion}-1.el7.src.rpm
 else
-    # just download the available RPMs
     mkdir -p /root/rpmbuild/RPMS/${arch}
     cd /root/rpmbuild/RPMS/${arch}
     wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-${cvmfsversion}-1.el7.${arch}.rpm

From 24dd67d865776a1935ef73f542d296ab5668e063 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 31 May 2021 17:37:39 +0200
Subject: [PATCH 19/56] install fuse-overlayfs

---
 containers/Dockerfile.EESSI-client-pilot-centos7 | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
index 443c5b90..e4564403 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -1,5 +1,5 @@
 ARG cvmfsversion=2.8.1
-
+ARG fuseoverlayfsversion=0.4.1
 
 FROM centos:7 AS prepare-rpm
 ARG cvmfsversion
@@ -7,10 +7,20 @@ COPY ./containers/build-or-download-cvmfs-rpms.sh /build-or-download-cvmfs-rpms.
 RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
 
 
+FROM centos:7 AS build-fuse-overlayfs
+ARG fuseoverlayfsversion
+RUN yum install -y wget fuse3-devel autoconf tar
+RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
+  && tar xzf v${fuseoverlayfsversion}.tar.gz \
+  && cd fuse-overlayfs-${fuseoverlayfsversion} \
+  && ./autogen.sh && ./configure && make && make install
+
+
 FROM centos:7
 ARG cvmfsversion
 
 COPY --from=prepare-rpm /root/rpmbuild/RPMS /root/rpmbuild/RPMS
+COPY --from=build-fuse-overlayfs /usr/local/bin/fuse-overlayfs /usr/local/bin/fuse-overlayfs
 
 RUN yum install -y sudo vim openssh-clients
 RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$(uname -m).rpm \

From 483166258cb76b95ae45ac31244161e706f0806e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 31 May 2021 17:40:04 +0200
Subject: [PATCH 20/56] add automake dependency

---
 containers/Dockerfile.EESSI-client-pilot-centos7 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
index e4564403..b9c44d42 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -9,7 +9,7 @@ RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
 
 FROM centos:7 AS build-fuse-overlayfs
 ARG fuseoverlayfsversion
-RUN yum install -y wget fuse3-devel autoconf tar
+RUN yum install -y wget fuse3-devel autoconf automake tar
 RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
   && tar xzf v${fuseoverlayfsversion}.tar.gz \
   && cd fuse-overlayfs-${fuseoverlayfsversion} \

From 97fc84164faf3e9b56996c95edf8adaab162ba63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 31 May 2021 17:42:45 +0200
Subject: [PATCH 21/56] add gcc compiler as dep

---
 containers/Dockerfile.EESSI-client-pilot-centos7 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
index b9c44d42..0502d729 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -9,7 +9,7 @@ RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
 
 FROM centos:7 AS build-fuse-overlayfs
 ARG fuseoverlayfsversion
-RUN yum install -y wget fuse3-devel autoconf automake tar
+RUN yum install -y wget fuse3-devel autoconf automake gcc tar
 RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
   && tar xzf v${fuseoverlayfsversion}.tar.gz \
   && cd fuse-overlayfs-${fuseoverlayfsversion} \

From 1a7e8ad3e3521a190430a79943b3ef0f1b89af2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Mon, 31 May 2021 17:49:50 +0200
Subject: [PATCH 22/56] add make as dep

---
 containers/Dockerfile.EESSI-client-pilot-centos7 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
index 0502d729..e64f4547 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -9,7 +9,7 @@ RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
 
 FROM centos:7 AS build-fuse-overlayfs
 ARG fuseoverlayfsversion
-RUN yum install -y wget fuse3-devel autoconf automake gcc tar
+RUN yum install -y wget fuse3-devel autoconf automake gcc make tar
 RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
   && tar xzf v${fuseoverlayfsversion}.tar.gz \
   && cd fuse-overlayfs-${fuseoverlayfsversion} \

From 804b8ad6843ade2aa14c4edebf4eeac68da0bcd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 1 Jun 2021 10:17:08 +0200
Subject: [PATCH 23/56] also publish containers on local branch for now

---
 .github/workflows/build-publish-containers.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 55d71717..725d89a4 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - use_latest_client_package
     paths:
       - containers/Dockerfile.EESSI-client*
       - containers/build-or-download-cvmfs-rpm.sh

From 24ed5822d9c0310f7ec465e37ce8d3c5313bea73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 1 Jun 2021 10:18:03 +0200
Subject: [PATCH 24/56] undo latest change

---
 .github/workflows/build-publish-containers.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 725d89a4..55d71717 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -3,7 +3,6 @@ on:
   push:
     branches:
       - main
-      - use_latest_client_package
     paths:
       - containers/Dockerfile.EESSI-client*
       - containers/build-or-download-cvmfs-rpm.sh

From e20cd98b789e65367657b0c838d6481586bd3320 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:28:22 +0200
Subject: [PATCH 25/56] use CVMFS_CLIENT_PROFILE

---
 containers/Dockerfile.EESSI-client-pilot-centos7 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
index e64f4547..90ead41b 100644
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -29,7 +29,7 @@ RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$
 RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
 
 RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_HTTP_PROXY="DIRECT"' >> /etc/cvmfs/default.local
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
 
 RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
 

From 428d38aa174f0c3ec3c4dd410dc58f25084ac762 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:28:39 +0200
Subject: [PATCH 26/56] first attempt for debian Dockerfile

---
 .../Dockerfile.EESSI-fuse-overlay-debian10    | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)
 create mode 100644 containers/Dockerfile.EESSI-fuse-overlay-debian10

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
new file mode 100644
index 00000000..51232125
--- /dev/null
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -0,0 +1,26 @@
+ARG cvmfsversion=2.8.1
+ARG fuseoverlayfsversion=0.4.1
+
+FROM debian:10.6 AS prepare-deb
+ARG cvmfsversion
+COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-rpms.sh
+RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
+
+
+FROM debian:10.6
+ARG cvmfsversion
+
+COPY --from=prepare-deb /root/deb /root/deb
+
+RUN apt-get install -y sudo vim openssh-client gawk autofs fuse curl attr uuid fuse3
+RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(uname -m).deb \
+            /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(uname -m).deb \
+            /root/deb/cvmfs-config-eessi_latest_all.deb \
+            /root/deb/cvmfs-config-eessi_latest_all.deb
+
+RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
+
+RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
+
+RUN useradd -ms /bin/bash eessi

From 15310cd847dada31d100b746409b848d3e4e06b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:28:55 +0200
Subject: [PATCH 27/56] script for making/fetching .deb packages

---
 containers/build-or-download-cvmfs-debs.sh | 24 ++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 containers/build-or-download-cvmfs-debs.sh

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
new file mode 100644
index 00000000..c46fb2e5
--- /dev/null
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -0,0 +1,24 @@
+cvmfsversion=$1
+arch=$(uname -m)
+
+yum install -y wget
+if [ "$arch" = "ppc64le" || "$arch" = "aarch64" ]
+then
+    apt-get install -y wget devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev
+    cd /tmp
+    wget http://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/source.tar.gz
+    wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
+    tar xzf cvmfs-${cvmfsversion}.tar.gz
+    cd cvmfs-${cvmfsversion}/ci/cvmfs    
+    mkdir /root/deb
+    ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb
+else
+    mkdir -p /root/deb
+    cd /root/deb
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs_${cvmfsversion}~1+debian10_${arch}.deb
+    wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/cvmfs-fuse3_${cvmfsversion}~1+debian10_${arch}.deb
+fi
+
+cd /root/deb
+wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default_latest_all.deb
+wget https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb

From b012e5d7e0e9f80229f5c330b3f3ba35d08e9913 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:33:34 +0200
Subject: [PATCH 28/56] rename Dockerfile so that they include the tag name

---
 .../Dockerfile.EESSI-client-pilot-centos7     | 36 -------------------
 .../Dockerfile.EESSI-fuse-overlay-debian10    | 26 --------------
 2 files changed, 62 deletions(-)
 delete mode 100644 containers/Dockerfile.EESSI-client-pilot-centos7
 delete mode 100644 containers/Dockerfile.EESSI-fuse-overlay-debian10

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
deleted file mode 100644
index 90ead41b..00000000
--- a/containers/Dockerfile.EESSI-client-pilot-centos7
+++ /dev/null
@@ -1,36 +0,0 @@
-ARG cvmfsversion=2.8.1
-ARG fuseoverlayfsversion=0.4.1
-
-FROM centos:7 AS prepare-rpm
-ARG cvmfsversion
-COPY ./containers/build-or-download-cvmfs-rpms.sh /build-or-download-cvmfs-rpms.sh
-RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
-
-
-FROM centos:7 AS build-fuse-overlayfs
-ARG fuseoverlayfsversion
-RUN yum install -y wget fuse3-devel autoconf automake gcc make tar
-RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
-  && tar xzf v${fuseoverlayfsversion}.tar.gz \
-  && cd fuse-overlayfs-${fuseoverlayfsversion} \
-  && ./autogen.sh && ./configure && make && make install
-
-
-FROM centos:7
-ARG cvmfsversion
-
-COPY --from=prepare-rpm /root/rpmbuild/RPMS /root/rpmbuild/RPMS
-COPY --from=build-fuse-overlayfs /usr/local/bin/fuse-overlayfs /usr/local/bin/fuse-overlayfs
-
-RUN yum install -y sudo vim openssh-clients
-RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$(uname -m).rpm \
-                   /root/rpmbuild/RPMS/$(uname -m)/cvmfs-fuse3-${cvmfsversion}-1.el7.$(uname -m).rpm \
-                   http://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default-latest.noarch.rpm
-RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
-
-RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
-
-RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
-
-RUN useradd -ms /bin/bash eessi
diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
deleted file mode 100644
index 51232125..00000000
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ /dev/null
@@ -1,26 +0,0 @@
-ARG cvmfsversion=2.8.1
-ARG fuseoverlayfsversion=0.4.1
-
-FROM debian:10.6 AS prepare-deb
-ARG cvmfsversion
-COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-rpms.sh
-RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
-
-
-FROM debian:10.6
-ARG cvmfsversion
-
-COPY --from=prepare-deb /root/deb /root/deb
-
-RUN apt-get install -y sudo vim openssh-client gawk autofs fuse curl attr uuid fuse3
-RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(uname -m).deb \
-            /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(uname -m).deb \
-            /root/deb/cvmfs-config-eessi_latest_all.deb \
-            /root/deb/cvmfs-config-eessi_latest_all.deb
-
-RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
-  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
-
-RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
-
-RUN useradd -ms /bin/bash eessi

From d802a432bf3ddb915baa649805b9707bf2ab0e88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:33:58 +0200
Subject: [PATCH 29/56] use matrix, also build fuse-overlay images

---
 .github/workflows/build-publish-containers.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 55d71717..adc7f5e7 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -26,6 +26,10 @@ jobs:
     permissions:
       packages: write
       contents: read
+    strategy:
+      matrix:
+        dockerfiles: [client-pilot:centos7, fuse-overlay:debian10]
+
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
@@ -59,8 +63,8 @@ jobs:
       - name: Build and push to GitHub Packages
         uses: docker/build-push-action@v2
         with:
-          tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/client-pilot:centos7
-          file: containers/Dockerfile.EESSI-client-pilot-centos7
+          tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}
+          file: containers/Dockerfile.EESSI-${{ matrix.tag }}
           platforms: linux/amd64, linux/arm64, linux/ppc64le
           push: ${{ github.event_name != 'pull_request' }}
           cache-from: type=local,src=/tmp/.buildx-cache

From b986164dc4f73226e071c844d2b9b24d477b4018 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:36:54 +0200
Subject: [PATCH 30/56] small changes to workflow

---
 .github/workflows/build-publish-containers.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index adc7f5e7..064b7f43 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -1,11 +1,11 @@
-name: Build and publish EESSI client Docker image
+name: Build and publish EESSI container images
 on:
   push:
     branches:
       - main
     paths:
-      - containers/Dockerfile.EESSI-client*
-      - containers/build-or-download-cvmfs-rpm.sh
+      - containers/Dockerfile.EESSI-*
+      - containers/build-or-download-cvmfs-*.sh
 
   pull_request:
     branches:
@@ -28,7 +28,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        dockerfiles: [client-pilot:centos7, fuse-overlay:debian10]
+        dockerfiles: ['client-pilot:centos7', 'fuse-overlay:debian10']
 
     steps:
       - name: Check out the repo

From 158ee10695ed4adbedc351db526764259756de32 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:39:12 +0200
Subject: [PATCH 31/56] fix matrix item name

---
 .github/workflows/build-publish-containers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 064b7f43..dd46670e 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -28,7 +28,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        dockerfiles: ['client-pilot:centos7', 'fuse-overlay:debian10']
+        tag: ['client-pilot:centos7', 'fuse-overlay:debian10']
 
     steps:
       - name: Check out the repo

From 7fa6e1736fe2dac91763deaba42431abf1f0edf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:42:36 +0200
Subject: [PATCH 32/56] use quotes around filename

---
 .github/workflows/build-publish-containers.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index dd46670e..e2502c35 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -64,7 +64,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}
-          file: containers/Dockerfile.EESSI-${{ matrix.tag }}
+          file: "containers/Dockerfile.EESSI-${{ matrix.tag }}"
           platforms: linux/amd64, linux/arm64, linux/ppc64le
           push: ${{ github.event_name != 'pull_request' }}
           cache-from: type=local,src=/tmp/.buildx-cache

From 259a1d62fc9ceffd5164b191bf025c603a1c9e47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:48:56 +0200
Subject: [PATCH 33/56] use bash to set name of Dockerfile without colons

---
 .github/workflows/build-publish-containers.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index e2502c35..639a8d20 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -56,15 +56,17 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Convert and store repository owner in lowercase
+      - name: Convert and store repository owner in lowercase, replace colon in tag names by hyphen
         run: |
           echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
+          tag=${{ matrix.tag }}
+          echo DOCKERFILE=Dockerfile.EESSI-${tag//:/-} >> $GITHUB_ENV
 
       - name: Build and push to GitHub Packages
         uses: docker/build-push-action@v2
         with:
           tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}
-          file: "containers/Dockerfile.EESSI-${{ matrix.tag }}"
+          file: containers/${{ env.DOCKERFILE }}
           platforms: linux/amd64, linux/arm64, linux/ppc64le
           push: ${{ github.event_name != 'pull_request' }}
           cache-from: type=local,src=/tmp/.buildx-cache

From c7fea4b6b9264a8753fa038b1b6ece028d7d8580 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:50:55 +0200
Subject: [PATCH 34/56] undo name change

---
 .../Dockerfile.EESSI-client-pilot-centos7     | 36 +++++++++++++++++++
 .../Dockerfile.EESSI-fuse-overlay-debian10    | 26 ++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 containers/Dockerfile.EESSI-client-pilot-centos7
 create mode 100644 containers/Dockerfile.EESSI-fuse-overlay-debian10

diff --git a/containers/Dockerfile.EESSI-client-pilot-centos7 b/containers/Dockerfile.EESSI-client-pilot-centos7
new file mode 100644
index 00000000..90ead41b
--- /dev/null
+++ b/containers/Dockerfile.EESSI-client-pilot-centos7
@@ -0,0 +1,36 @@
+ARG cvmfsversion=2.8.1
+ARG fuseoverlayfsversion=0.4.1
+
+FROM centos:7 AS prepare-rpm
+ARG cvmfsversion
+COPY ./containers/build-or-download-cvmfs-rpms.sh /build-or-download-cvmfs-rpms.sh
+RUN sh /build-or-download-cvmfs-rpms.sh ${cvmfsversion}
+
+
+FROM centos:7 AS build-fuse-overlayfs
+ARG fuseoverlayfsversion
+RUN yum install -y wget fuse3-devel autoconf automake gcc make tar
+RUN wget https://github.com/containers/fuse-overlayfs/archive/refs/tags/v${fuseoverlayfsversion}.tar.gz \
+  && tar xzf v${fuseoverlayfsversion}.tar.gz \
+  && cd fuse-overlayfs-${fuseoverlayfsversion} \
+  && ./autogen.sh && ./configure && make && make install
+
+
+FROM centos:7
+ARG cvmfsversion
+
+COPY --from=prepare-rpm /root/rpmbuild/RPMS /root/rpmbuild/RPMS
+COPY --from=build-fuse-overlayfs /usr/local/bin/fuse-overlayfs /usr/local/bin/fuse-overlayfs
+
+RUN yum install -y sudo vim openssh-clients
+RUN yum install -y /root/rpmbuild/RPMS/$(uname -m)/cvmfs-${cvmfsversion}-1.el7.$(uname -m).rpm \
+                   /root/rpmbuild/RPMS/$(uname -m)/cvmfs-fuse3-${cvmfsversion}-1.el7.$(uname -m).rpm \
+                   http://ecsft.cern.ch/dist/cvmfs/cvmfs-config/cvmfs-config-default-latest.noarch.rpm
+RUN yum install -y https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi-latest.noarch.rpm
+
+RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
+
+RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
+
+RUN useradd -ms /bin/bash eessi
diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
new file mode 100644
index 00000000..51232125
--- /dev/null
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -0,0 +1,26 @@
+ARG cvmfsversion=2.8.1
+ARG fuseoverlayfsversion=0.4.1
+
+FROM debian:10.6 AS prepare-deb
+ARG cvmfsversion
+COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-rpms.sh
+RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
+
+
+FROM debian:10.6
+ARG cvmfsversion
+
+COPY --from=prepare-deb /root/deb /root/deb
+
+RUN apt-get install -y sudo vim openssh-client gawk autofs fuse curl attr uuid fuse3
+RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(uname -m).deb \
+            /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(uname -m).deb \
+            /root/deb/cvmfs-config-eessi_latest_all.deb \
+            /root/deb/cvmfs-config-eessi_latest_all.deb
+
+RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \
+  && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local
+
+RUN mkdir -p /cvmfs/pilot.eessi-hpc.org
+
+RUN useradd -ms /bin/bash eessi

From 2ccb9d9cbea278b92a90bde2c200cff55a666336 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:54:09 +0200
Subject: [PATCH 35/56] fix typo

---
 containers/Dockerfile.EESSI-fuse-overlay-debian10 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
index 51232125..55832cc6 100644
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -3,7 +3,7 @@ ARG fuseoverlayfsversion=0.4.1
 
 FROM debian:10.6 AS prepare-deb
 ARG cvmfsversion
-COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-rpms.sh
+COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh
 RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion}
 
 

From ca9a550219fc6967b0b4f385ba69d58e722aa750 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 11:59:39 +0200
Subject: [PATCH 36/56] fix if statement, install wget

---
 containers/build-or-download-cvmfs-debs.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index c46fb2e5..6defdc59 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -1,10 +1,10 @@
 cvmfsversion=$1
 arch=$(uname -m)
 
-yum install -y wget
-if [ "$arch" = "ppc64le" || "$arch" = "aarch64" ]
+apt-get install -y wget
+if [ "$arch" = "ppc64le" ] || [ "$arch" = "aarch64" ]
 then
-    apt-get install -y wget devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev
+    apt-get install devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev
     cd /tmp
     wget http://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/source.tar.gz
     wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz

From b373a190d7877c72c005d56b220c9ec1f7871df3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 12:03:59 +0200
Subject: [PATCH 37/56] add an apt-get update command

---
 containers/build-or-download-cvmfs-debs.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index 6defdc59..6816ca35 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -1,6 +1,7 @@
 cvmfsversion=$1
 arch=$(uname -m)
 
+apt-get update
 apt-get install -y wget
 if [ "$arch" = "ppc64le" ] || [ "$arch" = "aarch64" ]
 then

From ef47a139ccf180b8229ab456ef2b0f73451841a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 12:07:41 +0200
Subject: [PATCH 38/56] add an apt-get update command

---
 containers/Dockerfile.EESSI-fuse-overlay-debian10 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
index 55832cc6..3b0d9b67 100644
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -12,6 +12,7 @@ ARG cvmfsversion
 
 COPY --from=prepare-deb /root/deb /root/deb
 
+RUN apt-get update
 RUN apt-get install -y sudo vim openssh-client gawk autofs fuse curl attr uuid fuse3
 RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(uname -m).deb \
             /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(uname -m).deb \

From 0f26e9da99d9064860361b83f7c15a63f4675632 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 12:12:30 +0200
Subject: [PATCH 39/56] dont install fuse in final container, only fuse3

---
 containers/Dockerfile.EESSI-fuse-overlay-debian10 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
index 3b0d9b67..abc11d46 100644
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -13,7 +13,7 @@ ARG cvmfsversion
 COPY --from=prepare-deb /root/deb /root/deb
 
 RUN apt-get update
-RUN apt-get install -y sudo vim openssh-client gawk autofs fuse curl attr uuid fuse3
+RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3
 RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(uname -m).deb \
             /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(uname -m).deb \
             /root/deb/cvmfs-config-eessi_latest_all.deb \

From d0293448fc1261088d559af869884d99bcc4f3c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 12:29:48 +0200
Subject: [PATCH 40/56] use dpkg --print-architecture instead of uname for deb
 packages

---
 containers/Dockerfile.EESSI-fuse-overlay-debian10 | 4 ++--
 containers/build-or-download-cvmfs-debs.sh        | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
index abc11d46..75c1d34c 100644
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -14,8 +14,8 @@ COPY --from=prepare-deb /root/deb /root/deb
 
 RUN apt-get update
 RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3
-RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(uname -m).deb \
-            /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(uname -m).deb \
+RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
+            /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
             /root/deb/cvmfs-config-eessi_latest_all.deb \
             /root/deb/cvmfs-config-eessi_latest_all.deb
 
diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index 6816ca35..fe15f58d 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -1,9 +1,9 @@
 cvmfsversion=$1
-arch=$(uname -m)
+arch=$(dpkg --print-architecture)
 
 apt-get update
 apt-get install -y wget
-if [ "$arch" = "ppc64le" ] || [ "$arch" = "aarch64" ]
+if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
     apt-get install devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev
     cd /tmp

From 40b7e0ef0608485785ac10ff7e442faa39e4be1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 12:41:16 +0200
Subject: [PATCH 41/56] remove duplicate entry of config package

---
 containers/Dockerfile.EESSI-fuse-overlay-debian10 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
index 75c1d34c..44f90a44 100644
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -16,7 +16,6 @@ RUN apt-get update
 RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3
 RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
             /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
-            /root/deb/cvmfs-config-eessi_latest_all.deb \
             /root/deb/cvmfs-config-eessi_latest_all.deb
 
 RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \

From 7e68248249da581104cafe5e1b2a76b078ef5e53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 12:41:59 +0200
Subject: [PATCH 42/56] add cvmfs-config-default

---
 containers/Dockerfile.EESSI-fuse-overlay-debian10 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
index 44f90a44..492a90bf 100644
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -16,6 +16,7 @@ RUN apt-get update
 RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3
 RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
             /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
+            /root/deb/cvmfs-config-default_latest_all.deb \
             /root/deb/cvmfs-config-eessi_latest_all.deb
 
 RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \

From d55222cb8b5cebfda9ca579ae7f4ff32aff5edf8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 12:45:32 +0200
Subject: [PATCH 43/56] add some more packages (cvmfs deps)

---
 containers/Dockerfile.EESSI-fuse-overlay-debian10 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-fuse-overlay-debian10
index 492a90bf..b816d7dc 100644
--- a/containers/Dockerfile.EESSI-fuse-overlay-debian10
+++ b/containers/Dockerfile.EESSI-fuse-overlay-debian10
@@ -13,7 +13,7 @@ ARG cvmfsversion
 COPY --from=prepare-deb /root/deb /root/deb
 
 RUN apt-get update
-RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3
+RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev
 RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
             /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian10_$(dpkg --print-architecture).deb \
             /root/deb/cvmfs-config-default_latest_all.deb \

From 6dec944ef40a2441db984b39f419ec4e467edb04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 13:15:13 +0200
Subject: [PATCH 44/56] remove redundant download, fix dir path to cvmfs

---
 containers/build-or-download-cvmfs-debs.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index fe15f58d..fe558deb 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -7,10 +7,9 @@ if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
     apt-get install devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev
     cd /tmp
-    wget http://ecsft.cern.ch/dist/cvmfs/cvmfs-${cvmfsversion}/source.tar.gz
     wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
     tar xzf cvmfs-${cvmfsversion}.tar.gz
-    cd cvmfs-${cvmfsversion}/ci/cvmfs    
+    cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs
     mkdir /root/deb
     ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb
 else

From 70a6e4c5ffe9bd55a964b6f5b4b41ea0bfe9bd0f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 13:27:28 +0200
Subject: [PATCH 45/56] add lsb-release on archs where we build from source

---
 containers/build-or-download-cvmfs-debs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index fe558deb..0064ec93 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -5,7 +5,7 @@ apt-get update
 apt-get install -y wget
 if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
-    apt-get install devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev
+    apt-get install devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
     cd /tmp
     wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
     tar xzf cvmfs-${cvmfsversion}.tar.gz

From 7d343756a89bb28d9d74f5502cfc3b886a0144d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 13:33:57 +0200
Subject: [PATCH 46/56] add -y to apt-get

---
 containers/build-or-download-cvmfs-debs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index 0064ec93..a0e3ccc4 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -5,7 +5,7 @@ apt-get update
 apt-get install -y wget
 if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
-    apt-get install devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
+    apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
     cd /tmp
     wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
     tar xzf cvmfs-${cvmfsversion}.tar.gz

From aa94aa0b2ce5f95d142308b817b85274f7481cd9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 13:42:28 +0200
Subject: [PATCH 47/56] debug missing lsb-release

---
 containers/build-or-download-cvmfs-debs.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index a0e3ccc4..32fd0fe7 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -6,6 +6,9 @@ apt-get install -y wget
 if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
     apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
+    apt list lsb-release
+    which lsb-release
+    ls -l /usr/bin/lsb-release
     cd /tmp
     wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
     tar xzf cvmfs-${cvmfsversion}.tar.gz

From e2688a0ab49e5b83409f5b2172662db6e348e3ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 14:15:34 +0200
Subject: [PATCH 48/56] add ppc64 arch

---
 containers/build-or-download-cvmfs-debs.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index 32fd0fe7..a531f597 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -5,6 +5,7 @@ apt-get update
 apt-get install -y wget
 if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
+    dpkg --add-architecture ${arch}
     apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
     apt list lsb-release
     which lsb-release

From 0a43968106eaf096456b12bdb8673c14677bc237 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 14:36:41 +0200
Subject: [PATCH 49/56] add ppc64el architecture to cvmfs control file

---
 containers/build-or-download-cvmfs-debs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index a531f597..7653e606 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -5,7 +5,6 @@ apt-get update
 apt-get install -y wget
 if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
-    dpkg --add-architecture ${arch}
     apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
     apt list lsb-release
     which lsb-release
@@ -15,6 +14,7 @@ then
     tar xzf cvmfs-${cvmfsversion}.tar.gz
     cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs
     mkdir /root/deb
+    sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 ppc64el/' ../../packaging/debian/cvmfs/control.in
     ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb
 else
     mkdir -p /root/deb

From 415ba22d7a6a7aac0078a4ea370e05b214718eff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 16:20:20 +0200
Subject: [PATCH 50/56] update README

---
 containers/README.md | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/containers/README.md b/containers/README.md
index faa898eb..302cfc04 100644
--- a/containers/README.md
+++ b/containers/README.md
@@ -6,7 +6,7 @@ This directory contains recipes for containers that are useful in the scope of t
 
 Container to provide easy access to EESSI pilot repository,
 see https://github.com/users/EESSI/packages/container/package/client-pilot and https://eessi.github.io/docs/pilot.
-This container image gets automatically built and pushed to the GitHub Container Registry when one of its
+This container image is based on CentOS 7, and gets automatically built and pushed to the GitHub Container Registry when one of its
 source files (the Dockerfile or the script that generates the CernVM-FS RPMs) gets changed,
 or when a new version of the filesystem-layer repository is released.
 
@@ -19,3 +19,27 @@ export SINGULARITY_HOME="/tmp/$USER/home:/home/$USER"
 export EESSI_PILOT="container:cvmfs2 pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org"
 singularity shell --fusemount "$EESSI_PILOT" docker://ghcr.io/EESSI/client-pilot:centos7
 ```
+
+## Fuse-overlayfs container
+
+Container that can be used to build and install software to /cvmfs by leveraging fuse-overlayfs for
+providing a writable overlay.
+The container image is based on Debian 10.6, and gets automatically built and pushed to the GitHub Container Registry when one of its
+source files (the Dockerfile or the script that generates the CernVM-FS deb packages) gets changed,
+or when a new version of the filesystem-layer repository is released.
+
+### Run (using Singularity)
+```
+export EESSI_TMPDIR=/tmp/$USER/EESSI
+mkdir -p $EESSI_TMPDIR
+mkdir -p $EESSI_TMPDIR/{home,overlay-upper,overlay-work}
+mkdir -p $EESSI_TMPDIR/{var-lib-cvmfs,var-run-cvmfs}
+export SINGULARITY_CACHEDIR=$EESSI_TMPDIR/singularity_cache
+export SINGULARITY_BIND="$EESSI_TMPDIR/var-run-cvmfs:/var/run/cvmfs,$EESSI_TMPDIR/var-lib-cvmfs:/var/lib/cvmfs"
+export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER"
+export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org"
+export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org"
+singularity shell --fusemount "$EESSI_PILOT_READONLY" --fusemount "$EESSI_PILOT_WRITABLE_OVERLAY" docker://eessi/fuse-overlay:debian10-$(uname -m)
+```
+
+For more details about building software, see: https://eessi.github.io/docs/software_layer/build_nodes/

From ca0bed226faab7621f2805c559502a39e75d5e3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 4 Jun 2021 16:23:02 +0200
Subject: [PATCH 51/56] remove redundant lines

---
 containers/build-or-download-cvmfs-debs.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh
index 7653e606..cb514298 100644
--- a/containers/build-or-download-cvmfs-debs.sh
+++ b/containers/build-or-download-cvmfs-debs.sh
@@ -6,9 +6,6 @@ apt-get install -y wget
 if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ]
 then
     apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools uuid-dev valgrind libz-dev lsb-release
-    apt list lsb-release
-    which lsb-release
-    ls -l /usr/bin/lsb-release
     cd /tmp
     wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz
     tar xzf cvmfs-${cvmfsversion}.tar.gz

From b34ca2f4e131d1dab378418089c557263f51ff45 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 23 Jun 2021 14:05:52 +0200
Subject: [PATCH 52/56] build for all archs seperately, build manifest at end

---
 .../workflows/build-publish-containers.yml    | 37 +++++++++++++++++--
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 639a8d20..e65545a8 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -29,6 +29,7 @@ jobs:
     strategy:
       matrix:
         tag: ['client-pilot:centos7', 'fuse-overlay:debian10']
+        platform: [amd64, arm64, ppc64le]
 
     steps:
       - name: Check out the repo
@@ -44,9 +45,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          key: ${{ runner.os }}-buildx-${{ matrix.platform }}-${{ matrix.tag }}
           restore-keys: |
-            ${{ runner.os }}-buildx-
+            ${{ runner.os }}-buildx-${{ matrix.platform }}-${{ matrix.tag }}
 
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
@@ -65,9 +66,9 @@ jobs:
       - name: Build and push to GitHub Packages
         uses: docker/build-push-action@v2
         with:
-          tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}
+          tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-${{ matrix.platform }}
           file: containers/${{ env.DOCKERFILE }}
-          platforms: linux/amd64, linux/arm64, linux/ppc64le
+          platforms: linux/${{ matrix.platform }}
           push: ${{ github.event_name != 'pull_request' }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache-new
@@ -76,3 +77,31 @@ jobs:
         run: |
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+  create_manifest:
+    name: Create the Docker manifest
+    needs: build_and_publish
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    strategy:
+      matrix:
+        tag: ['client-pilot:centos7', 'fuse-overlay:debian10']
+    steps:
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run docker manifest
+        run: |
+          export REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')
+          docker manifest create \
+            ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }} \
+            --amend ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}-amd64 \
+            --amend ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}-arm64 \
+            --amend ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}-ppc64le
+          docker manifest push ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}

From 8b151ba0c78e5509a6214fadfc1e0d709f7568b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 23 Jun 2021 14:18:29 +0200
Subject: [PATCH 53/56] only push manifest when this is not a PR

---
 .../workflows/build-publish-containers.yml    | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index e65545a8..a6b1884c 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -90,18 +90,25 @@ jobs:
         tag: ['client-pilot:centos7', 'fuse-overlay:debian10']
     steps:
       - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Convert and store repository owner in lowercase
+        run: |
+          echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
+
       - name: Run docker manifest
         run: |
-          export REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')
           docker manifest create \
-            ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }} \
-            --amend ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}-amd64 \
-            --amend ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}-arm64 \
-            --amend ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}-ppc64le
-          docker manifest push ghcr.io/${REPOSITORY_OWNER}/${{ matrix.tag }}
+            ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \
+            --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \
+            --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \
+            --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-ppc64le
+
+      - name: Push manifest
+        if: github.event_name != 'pull_request'
+        run: docker manifest push ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}

From d74fa2db7dd1ce0dcf1b08b7dd982703a01f6bf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 23 Jun 2021 22:00:58 +0200
Subject: [PATCH 54/56] only create manifest when not running in PR

---
 .github/workflows/build-publish-containers.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index a6b1884c..3ba3836a 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -102,6 +102,7 @@ jobs:
           echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
 
       - name: Run docker manifest
+        if: github.event_name != 'pull_request'
         run: |
           docker manifest create \
             ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \

From 29e365af346fc0e1d6376ce95247db228678e426 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 24 Jun 2021 14:08:28 +0200
Subject: [PATCH 55/56] rename fuse-overlay-debian10 to build-node-debian10

---
 .github/workflows/build-publish-containers.yml                | 4 ++--
 ...-overlay-debian10 => Dockerfile.EESSI-build-node-debian10} | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename containers/{Dockerfile.EESSI-fuse-overlay-debian10 => Dockerfile.EESSI-build-node-debian10} (100%)

diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml
index 3ba3836a..32d4c8ec 100644
--- a/.github/workflows/build-publish-containers.yml
+++ b/.github/workflows/build-publish-containers.yml
@@ -28,7 +28,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        tag: ['client-pilot:centos7', 'fuse-overlay:debian10']
+        tag: ['client-pilot:centos7', 'build-node:debian10']
         platform: [amd64, arm64, ppc64le]
 
     steps:
@@ -87,7 +87,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        tag: ['client-pilot:centos7', 'fuse-overlay:debian10']
+        tag: ['client-pilot:centos7', 'build-node:debian10']
     steps:
       - name: Login to GitHub Container Registry
         if: github.event_name != 'pull_request'
diff --git a/containers/Dockerfile.EESSI-fuse-overlay-debian10 b/containers/Dockerfile.EESSI-build-node-debian10
similarity index 100%
rename from containers/Dockerfile.EESSI-fuse-overlay-debian10
rename to containers/Dockerfile.EESSI-build-node-debian10

From 9abe22775b07bc5fccd15b65d84493cacf0bd76a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 24 Jun 2021 14:14:11 +0200
Subject: [PATCH 56/56] renamed fuse-overlay container to build-node

---
 containers/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/containers/README.md b/containers/README.md
index 302cfc04..fd857732 100644
--- a/containers/README.md
+++ b/containers/README.md
@@ -20,9 +20,9 @@ export EESSI_PILOT="container:cvmfs2 pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.
 singularity shell --fusemount "$EESSI_PILOT" docker://ghcr.io/EESSI/client-pilot:centos7
 ```
 
-## Fuse-overlayfs container
+## Build node container
 
-Container that can be used to build and install software to /cvmfs by leveraging fuse-overlayfs for
+Container that can be used to build and install software to /cvmfs by leveraging `fuse-overlayfs` for
 providing a writable overlay.
 The container image is based on Debian 10.6, and gets automatically built and pushed to the GitHub Container Registry when one of its
 source files (the Dockerfile or the script that generates the CernVM-FS deb packages) gets changed,