From 13157e765b695a1bb9c9e2e30dc4c8f6f0777626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 15 Sep 2022 13:15:06 +0200 Subject: [PATCH 01/18] add risc64 --- .github/workflows/build-publish-containers.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index 3f029979..86bb1dec 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -18,7 +18,7 @@ on: # this often means that we have a new CVMFS configuration package release: types: [published] - + jobs: build_and_publish: name: Build and publish image @@ -30,7 +30,7 @@ jobs: strategy: matrix: tag: ['client-pilot:centos7', 'build-node:debian10'] - platform: [amd64, arm64, ppc64le] + platform: [amd64, arm64, ppc64le, riscv64] steps: - name: Check out the repo @@ -109,7 +109,8 @@ jobs: ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \ --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \ --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \ - --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-ppc64le + --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-ppc64le \ + --amend ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-riscv64 - name: Push manifest if: github.event_name != 'pull_request' From 32fe776c7d7d1562d253c1e90d62f9e3b4fb8f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 15 Sep 2022 13:17:08 +0200 Subject: [PATCH 02/18] add riscv64 --- containers/build-or-download-cvmfs-debs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index 9f361698..03fed450 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -3,7 +3,7 @@ arch=$(dpkg --print-architecture) apt-get update apt-get install -y wget -if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ] +if [ "$arch" = "ppc64el" ] || [ "$arch" = "arm64" ] || [ "$arch" = "riscv64" ] then apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools python3-dev python3-setuptools uuid-dev valgrind libz-dev lsb-release # Set Python 2 as default Python @@ -14,7 +14,7 @@ then tar xzf cvmfs-${cvmfsversion}.tar.gz cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs mkdir /root/deb - sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 ppc64el/' ../../packaging/debian/cvmfs/control.in + sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 ppc64el riscv64/' ../../packaging/debian/cvmfs/control.in ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb else mkdir -p /root/deb From 16cab69742904709fede79666db3eda1e0144996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 15 Sep 2022 13:19:25 +0200 Subject: [PATCH 03/18] add riscv64 --- containers/build-or-download-cvmfs-rpms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/build-or-download-cvmfs-rpms.sh b/containers/build-or-download-cvmfs-rpms.sh index 078c4f71..feb29b7b 100644 --- a/containers/build-or-download-cvmfs-rpms.sh +++ b/containers/build-or-download-cvmfs-rpms.sh @@ -2,7 +2,7 @@ cvmfsversion=$1 arch=$(uname -m) yum install -y wget -if [ "$arch" = "ppc64le" ] +if [ "$arch" = "ppc64le" ] || [ "$arch" = "riscv64" ] then yum install -y epel-release yum install -y rpm-build checkpolicy cmake fuse-devel fuse3-devel gcc gcc-c++ golang libattr-devel libcap-devel libuuid-devel openssl-devel python-devel python-setuptools python3-devel python3-setuptools selinux-policy-devel valgrind-devel hardlink selinux-policy-targeted From 5bf780b2f9879469c3676d17fc85495812774ddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 15 Sep 2022 13:23:29 +0200 Subject: [PATCH 04/18] also rebuild when deb script is changed --- .github/workflows/build-publish-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index 86bb1dec..6c71aaa7 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -12,7 +12,7 @@ on: - main paths: - containers/Dockerfile.EESSI-* - - containers/build-or-download-cvmfs-rpm.sh + - containers/build-or-download-cvmfs-*.sh # also rebuild the containers for new releases of filesystem-layer: # this often means that we have a new CVMFS configuration package From a0987ca03f5e92ac32690fa4b0fdd19cfcfe31c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 14:52:44 +0200 Subject: [PATCH 05/18] add Dockerfile for Debian sid --- .../Dockerfile.EESSI-build-node-debian-sid | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 containers/Dockerfile.EESSI-build-node-debian-sid diff --git a/containers/Dockerfile.EESSI-build-node-debian-sid b/containers/Dockerfile.EESSI-build-node-debian-sid new file mode 100644 index 00000000..1bff6fd1 --- /dev/null +++ b/containers/Dockerfile.EESSI-build-node-debian-sid @@ -0,0 +1,48 @@ +ARG cvmfsversion=2.11.2 +ARG archspecversion=0.2.2 +ARG awscliversion=1.32.22 +ARG fuseoverlayfsversion=1.10 + +FROM debian:sid-20240330-slim AS prepare-deb +ARG cvmfsversion +COPY ./containers/build-or-download-cvmfs-debs.sh /build-or-download-cvmfs-debs.sh +RUN sh /build-or-download-cvmfs-debs.sh ${cvmfsversion} + + +FROM debian:sid-20240330-slim +ARG cvmfsversion +ARG archspecversion +ARG awscliversion +ARG fuseoverlayfsversion + +COPY --from=prepare-deb /root/deb /root/deb + +RUN apt-get update +RUN apt-get install -y sudo vim openssh-client gawk autofs curl attr uuid fuse3 libfuse2 psmisc gdb uuid-dev lsof strace +# python3 and jq are required for eessi-upload-to-staging script (next to awscli) +RUN apt-get install -y python3-pip python3-venv jq +RUN dpkg -i /root/deb/cvmfs_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-fuse3_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-libs_${cvmfsversion}~1+debian13_$(dpkg --print-architecture).deb \ + /root/deb/cvmfs-config-default_latest_all.deb \ + /root/deb/cvmfs-config-eessi_latest_all.deb + +# download binary for specific version of fuse-overlayfs +RUN curl -L -o /usr/local/bin/fuse-overlayfs https://github.com/containers/fuse-overlayfs/releases/download/v${fuseoverlayfsversion}/fuse-overlayfs-$(uname -m) \ + && chmod +x /usr/local/bin/fuse-overlayfs + +RUN echo 'CVMFS_QUOTA_LIMIT=10000' > /etc/cvmfs/default.local \ + && echo 'CVMFS_CLIENT_PROFILE="single"' >> /etc/cvmfs/default.local \ + && echo 'CVMFS_HIDE_MAGIC_XATTRS=yes' >> /etc/cvmfs/default.local + +RUN mkdir -p /cvmfs/software.eessi.io + +RUN useradd -ms /bin/bash eessi + +RUN python3 -m venv /opt/archspec && . /opt/archspec/bin/activate && pip3 install archspec==${archspecversion} && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec +# stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) +RUN python3 -m venv /opt/awscli && . /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws + +RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ + && mv eessi-upload-to-staging /usr/bin \ + && chmod a+x /usr/bin/eessi-upload-to-staging From 84c808e82eae6ffcabd519ef27bb6087dfe2c0c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 14:53:48 +0200 Subject: [PATCH 06/18] support multiple debian versions, add risc-v support --- containers/build-or-download-cvmfs-debs.sh | 33 +++++++++++++++++----- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index 42c860f4..1fab4331 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -1,22 +1,41 @@ #/bin/bash cvmfsversion=$1 arch=$(dpkg --print-architecture) -os=debian11 apt-get update -apt-get install -y wget +apt-get install -y wget lsb-release + +distro=$(lsb_release -si | tr [:upper:] [:lower:]) +release=$(lsb_release -sr) + +# lsb_release -sr prints n/a for debian sid, replace it by 13 +if [[ ${distro} = "debian" ]] && [[ ${release} = "n/a" ]] +then + release=13 +fi + +os="${distro}${release}" + if [ "$arch" = "arm64" ] || [ "$arch" = "riscv64" ] then - apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python-dev python-setuptools python3-dev python3-setuptools uuid-dev valgrind libz-dev lsb-release - # Set Python 2 as default Python - update-alternatives --install /usr/bin/python python /usr/bin/python2 1 - update-alternatives --install /usr/bin/python python /usr/bin/python3 2 + apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python3-dev python3-setuptools python3-dev python3-setuptools uuid-dev libz-dev lsb-release cd /tmp wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz tar xzf cvmfs-${cvmfsversion}.tar.gz cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs mkdir /root/deb - sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 ppc64el riscv64/' ../../packaging/debian/cvmfs/control.in + sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 riscv64/' ../../packaging/debian/cvmfs/control.in + sed -i 's/python-dev/python3-dev/' ../../packaging/debian/cvmfs/control.in + sed -i 's/python-setuptools/python3-setuptools/' ../../packaging/debian/cvmfs/control.in + # valgrind is not available (yet) for RISC-V + if [[ "$arch" = "riscv64" ]] + then + sed -i 's/, valgrind//' ../../packaging/debian/cvmfs/control.in + else + apt-get install -y valgrind + fi + # make sure the cvmfs package also uses debian 13 for debian sid + [[ $release = "n/a" ]] && sed "s@\$(lsb_release -sr)@13@" ./deb.sh ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb else mkdir -p /root/deb From 7b492a74d24705d8bedae4b128c47d1784c00642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 14:54:43 +0200 Subject: [PATCH 07/18] add debian sid container --- .github/workflows/build-publish-containers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index 99c37911..d57a2db5 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -91,7 +91,7 @@ jobs: contents: read strategy: matrix: - tag: ['client:centos7', 'build-node:debian11'] + tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] steps: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' From 628c7e0c09bf8b34416911afad7fd955e17a0e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 15:22:01 +0200 Subject: [PATCH 08/18] for now, only build debian sid --- .github/workflows/build-publish-containers.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index d57a2db5..528bb51d 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -32,7 +32,8 @@ jobs: contents: read strategy: matrix: - tag: ['client:centos7', 'build-node:debian11'] + #tag: ['client:centos7', 'build-node:debian11'] + tag: ['build-node:debian-sid'] platform: [amd64, arm64, riscv64] steps: @@ -91,7 +92,8 @@ jobs: contents: read strategy: matrix: - tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] + #tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] + tag: ['build-node:debian-sid'] steps: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' From 488d333e16023d2e22acd9e567d4c7c268b303eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 16:16:10 +0200 Subject: [PATCH 09/18] always build from source for debian13 --- containers/build-or-download-cvmfs-debs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index 1fab4331..d2c70b1f 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -16,7 +16,7 @@ fi os="${distro}${release}" -if [ "$arch" = "arm64" ] || [ "$arch" = "riscv64" ] +if [ "$arch" = "arm64" ] || [ "$arch" = "riscv64" ] || [ "${os}" = "debian13" ] then apt-get install -y devscripts libfuse3-dev cmake cpio libcap-dev libssl-dev libfuse-dev pkg-config libattr1-dev python3-dev python3-setuptools python3-dev python3-setuptools uuid-dev libz-dev lsb-release cd /tmp From fdd9d2c170bd38e274e4ef9803ede1cf0f6333b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 16:26:53 +0200 Subject: [PATCH 10/18] fix release check --- containers/build-or-download-cvmfs-debs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index d2c70b1f..d1ef8fbc 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -9,7 +9,7 @@ distro=$(lsb_release -si | tr [:upper:] [:lower:]) release=$(lsb_release -sr) # lsb_release -sr prints n/a for debian sid, replace it by 13 -if [[ ${distro} = "debian" ]] && [[ ${release} = "n/a" ]] +if [ "${distro}" = "debian" ] && [ "${release}" = "n/a" ] then release=13 fi From 8ef70500f01a77200178ee8f3b0f542236140839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 16:32:32 +0200 Subject: [PATCH 11/18] single square brackets --- containers/build-or-download-cvmfs-debs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index d1ef8fbc..7062240d 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -28,14 +28,14 @@ then sed -i 's/python-dev/python3-dev/' ../../packaging/debian/cvmfs/control.in sed -i 's/python-setuptools/python3-setuptools/' ../../packaging/debian/cvmfs/control.in # valgrind is not available (yet) for RISC-V - if [[ "$arch" = "riscv64" ]] + if [ "$arch" = "riscv64" ] then sed -i 's/, valgrind//' ../../packaging/debian/cvmfs/control.in else apt-get install -y valgrind fi # make sure the cvmfs package also uses debian 13 for debian sid - [[ $release = "n/a" ]] && sed "s@\$(lsb_release -sr)@13@" ./deb.sh + [ $release = "n/a" ] && sed "s@\$(lsb_release -sr)@13@" ./deb.sh ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb else mkdir -p /root/deb From fe704c01d89b62ac4315e6bca585a6211a758c63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 16:39:54 +0200 Subject: [PATCH 12/18] use correct check for cvmfs distro version --- containers/build-or-download-cvmfs-debs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index 7062240d..fe31e4ce 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -35,7 +35,7 @@ then apt-get install -y valgrind fi # make sure the cvmfs package also uses debian 13 for debian sid - [ $release = "n/a" ] && sed "s@\$(lsb_release -sr)@13@" ./deb.sh + [ $release = "13" ] && sed "s@\$(lsb_release -sr)@13@" ./deb.sh ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb else mkdir -p /root/deb From 47b44ed896237aa1dd025d03465bfed132b91781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 5 Apr 2024 16:44:29 +0200 Subject: [PATCH 13/18] add -i to sed --- containers/build-or-download-cvmfs-debs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index fe31e4ce..cc7a0954 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -35,7 +35,7 @@ then apt-get install -y valgrind fi # make sure the cvmfs package also uses debian 13 for debian sid - [ $release = "13" ] && sed "s@\$(lsb_release -sr)@13@" ./deb.sh + [ $release = "13" ] && sed -i "s@\$(lsb_release -sr)@13@" ./deb.sh ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb else mkdir -p /root/deb From 25022cd9651936f88543983c8c13657847930520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 8 Apr 2024 20:34:10 +0200 Subject: [PATCH 14/18] make sure fuse is properly handled on trixie --- containers/build-or-download-cvmfs-debs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index cc7a0954..213007df 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -35,7 +35,7 @@ then apt-get install -y valgrind fi # make sure the cvmfs package also uses debian 13 for debian sid - [ $release = "13" ] && sed -i "s@\$(lsb_release -sr)@13@" ./deb.sh + [ $release = "13" ] && sed -i "s@\$(lsb_release -sr)@13@" ./deb.sh && sed -i "s/focal/trixie/" ./deb.sh ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb else mkdir -p /root/deb From 40632267c264e6847851630b896a3477dab08d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 9 Apr 2024 13:11:22 +0200 Subject: [PATCH 15/18] use different start dir, add patch from PR 3446 for RISC-V --- containers/build-or-download-cvmfs-debs.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index 213007df..e680e1b5 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -22,18 +22,24 @@ then cd /tmp wget https://github.com/cvmfs/cvmfs/archive/refs/tags/cvmfs-${cvmfsversion}.tar.gz tar xzf cvmfs-${cvmfsversion}.tar.gz - cd cvmfs-cvmfs-${cvmfsversion}/ci/cvmfs + cd cvmfs-cvmfs-${cvmfsversion} mkdir /root/deb - sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 riscv64/' ../../packaging/debian/cvmfs/control.in - sed -i 's/python-dev/python3-dev/' ../../packaging/debian/cvmfs/control.in - sed -i 's/python-setuptools/python3-setuptools/' ../../packaging/debian/cvmfs/control.in - # valgrind is not available (yet) for RISC-V + sed -i 's/Architecture: i386 amd64 armhf arm64/Architecture: i386 amd64 armhf arm64 riscv64/' packaging/debian/cvmfs/control.in + sed -i 's/python-dev/python3-dev/' packaging/debian/cvmfs/control.in + sed -i 's/python-setuptools/python3-setuptools/' packaging/debian/cvmfs/control.in if [ "$arch" = "riscv64" ] then - sed -i 's/, valgrind//' ../../packaging/debian/cvmfs/control.in + # valgrind is not available (yet) for RISC-V + sed -i 's/, valgrind//' packaging/debian/cvmfs/control.in + # for RISC-V we need to run autoreconf, see: + # https://github.com/cvmfs/cvmfs/pull/3446 + wget https://github.com/cvmfs/cvmfs/pull/3446.patch + patch -p 1 -i ./3446.patch + rm 3446.patch else apt-get install -y valgrind fi + cd ci/cvmfs # make sure the cvmfs package also uses debian 13 for debian sid [ $release = "13" ] && sed -i "s@\$(lsb_release -sr)@13@" ./deb.sh && sed -i "s/focal/trixie/" ./deb.sh ./deb.sh /tmp/cvmfs-cvmfs-${cvmfsversion} /root/deb From 42aa1d6d3f326a7dc0c188394c61370f00c97bf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 12 Apr 2024 10:09:08 +0200 Subject: [PATCH 16/18] use uname -m for cpu detection on risc-v --- containers/build-or-download-cvmfs-debs.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/containers/build-or-download-cvmfs-debs.sh b/containers/build-or-download-cvmfs-debs.sh index e680e1b5..cd81bee9 100644 --- a/containers/build-or-download-cvmfs-debs.sh +++ b/containers/build-or-download-cvmfs-debs.sh @@ -36,6 +36,11 @@ then wget https://github.com/cvmfs/cvmfs/pull/3446.patch patch -p 1 -i ./3446.patch rm 3446.patch + # QEMU shows the host CPU in /proc/cpuinfo, so we need to tweak the CPU detection for some packages and use uname -m instead + sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/libcrypto/src/configureHook.sh + sed -i "s/rv64/riscv64/" externals/libcrypto/src/configureHook.sh + sed -i "s/^ISA=.*/ISA=\$(uname -m)/" externals/protobuf/src/configureHook.sh + sed -i "s/rv64/riscv64/" externals/protobuf/src/configureHook.sh else apt-get install -y valgrind fi From e0e8c5915c36032936e6af7cbd714260f9e5c8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Sun, 14 Apr 2024 10:28:51 +0200 Subject: [PATCH 17/18] build all images, only build for risc-v for debian sid images --- .../workflows/build-publish-containers.yml | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-publish-containers.yml b/.github/workflows/build-publish-containers.yml index 528bb51d..a7c9b607 100644 --- a/.github/workflows/build-publish-containers.yml +++ b/.github/workflows/build-publish-containers.yml @@ -32,9 +32,14 @@ jobs: contents: read strategy: matrix: - #tag: ['client:centos7', 'build-node:debian11'] - tag: ['build-node:debian-sid'] + tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] platform: [amd64, arm64, riscv64] + exclude: + # exclude images that don't support RISC-V + - tag: client:centos7 + platform: riscv64 + - tag: build-node:debian11 + platform: riscv64 steps: - name: Check out the repo @@ -92,8 +97,7 @@ jobs: contents: read strategy: matrix: - #tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] - tag: ['build-node:debian-sid'] + tag: ['client:centos7', 'build-node:debian11', 'build-node:debian-sid'] steps: - name: Login to GitHub Container Registry if: github.event_name != 'pull_request' @@ -108,13 +112,20 @@ jobs: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV - name: Create the final image using "docker buildx imagetools create" - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && matrix.tag == 'build-node:debian-sid' run: | docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \ ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \ ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 \ ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-riscv64 + - name: Create the final image using "docker buildx imagetools create" + if: github.event_name != 'pull_request' && matrix.tag != 'build-node:debian-sid' + run: | + docker buildx imagetools create -t ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} \ + ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-amd64 \ + ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }}-arm64 + - name: Inspect the final image if: github.event_name != 'pull_request' run: docker buildx imagetools inspect ghcr.io/${{ env.REPOSITORY_OWNER }}/${{ matrix.tag }} From 8da6758a033e8dc44d781a3b83bfad25a20470b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Apr 2024 12:35:42 +0200 Subject: [PATCH 18/18] remove eessi-upload-to-staging script from image --- containers/Dockerfile.EESSI-build-node-debian-sid | 4 ---- containers/Dockerfile.EESSI-build-node-debian11 | 4 ---- 2 files changed, 8 deletions(-) diff --git a/containers/Dockerfile.EESSI-build-node-debian-sid b/containers/Dockerfile.EESSI-build-node-debian-sid index 1bff6fd1..864919e5 100644 --- a/containers/Dockerfile.EESSI-build-node-debian-sid +++ b/containers/Dockerfile.EESSI-build-node-debian-sid @@ -42,7 +42,3 @@ RUN useradd -ms /bin/bash eessi RUN python3 -m venv /opt/archspec && . /opt/archspec/bin/activate && pip3 install archspec==${archspecversion} && deactivate && ln -s /opt/archspec/bin/archspec /usr/local/bin/archspec # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) RUN python3 -m venv /opt/awscli && . /opt/awscli/bin/activate && pip3 install awscli==${awscliversion} && deactivate && ln -s /opt/awscli/bin/aws /usr/local/bin/aws - -RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ - && mv eessi-upload-to-staging /usr/bin \ - && chmod a+x /usr/bin/eessi-upload-to-staging diff --git a/containers/Dockerfile.EESSI-build-node-debian11 b/containers/Dockerfile.EESSI-build-node-debian11 index 4fe34ae3..7b20a8e8 100644 --- a/containers/Dockerfile.EESSI-build-node-debian11 +++ b/containers/Dockerfile.EESSI-build-node-debian11 @@ -39,7 +39,3 @@ RUN useradd -ms /bin/bash eessi # stick to awscli v1.x, 2.x is not available through PyPI (see https://github.com/aws/aws-cli/issues/4947) RUN pip3 install archspec awscli==${awscliversion} - -RUN curl -OL https://raw.githubusercontent.com/EESSI/eessi-bot-software-layer/develop/scripts/eessi-upload-to-staging \ - && mv eessi-upload-to-staging /usr/bin \ - && chmod a+x /usr/bin/eessi-upload-to-staging