From 519d59106182d0f7d73bacad086314b8ba70a4b1 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 9 Sep 2025 11:45:41 +0200 Subject: [PATCH 01/36] Update trusted directories for glibc to leverage variable symlinks Fixes #226 Since this is pretty relevant to security, I am inclined to point these variable symlinks to `/dev/null` by default but that does not actually address the problem being discussed in #226 (having to harass the admins to link the CUDA drivers). If we can have logic in our CVMFS configuration then maybe we can address that. --- .../playbooks/roles/compatibility_layer/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index 9c7aab84..ddd867db 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -20,9 +20,9 @@ gentoo_git_repo: https://github.com/gentoo/gentoo.git gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7 prefix_required_space: 15 GB prefix_user_defined_trusted_dirs: - - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" - - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" - - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" prefix_mask_packages: | # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, # see for example https://github.com/EESSI/software-layer/issues/151 From 86f27d5e00336da59445d8c16e9b6276d2666b4e Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 9 Sep 2025 12:27:24 +0200 Subject: [PATCH 02/36] Update `upload-artifact` action --- .github/workflows/scorecards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 39b06bd8..8200819f 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -62,7 +62,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: SARIF file path: results.sarif From 561194d2b398d1816052023dcc70e7257b034953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 19:59:30 +0200 Subject: [PATCH 03/36] only sync when the overlay config is changed --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index f728896b..e642ae54 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -17,6 +17,7 @@ dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/{{ item.name }}.conf" mode: "0644" loop: "{{ custom_overlays }}" + register: overlay_config_files - name: Make configuration file with overlays that can override eclasses ansible.builtin.copy: @@ -33,6 +34,7 @@ community.general.portage: sync: 'yes' verbose: true + when: overlay_config_files is changed - name: Find all files and directories in the etc/portage directory of the overlay ansible.builtin.find: From ba11c639c2f66e2cbd43816ac1c29838fc5944c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:32:52 +0200 Subject: [PATCH 04/36] add sync handler --- .../playbooks/roles/compatibility_layer/handlers/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml index 15f88ef7..0780ff59 100644 --- a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml @@ -4,3 +4,9 @@ - name: Generate locales ansible.builtin.command: locale-gen changed_when: true + +- name: Sync the overlay repositories + community.general.portage: + sync: 'yes' + verbose: true + From 63f0dd4469cc8e0f6202fbf8c3bd572565e5328f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:34:08 +0200 Subject: [PATCH 05/36] rename sync handler --- ansible/playbooks/roles/compatibility_layer/handlers/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml index 0780ff59..195b1912 100644 --- a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml @@ -5,7 +5,7 @@ ansible.builtin.command: locale-gen changed_when: true -- name: Sync the overlay repositories +- name: Sync overlays community.general.portage: sync: 'yes' verbose: true From 4f95707162e7b8acf4ae6430226b0fb381217a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:34:21 +0200 Subject: [PATCH 06/36] call sync handler --- .../roles/compatibility_layer/tasks/add_overlay.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index e642ae54..16f75e0b 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -17,7 +17,7 @@ dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/{{ item.name }}.conf" mode: "0644" loop: "{{ custom_overlays }}" - register: overlay_config_files + notify: Sync overlays - name: Make configuration file with overlays that can override eclasses ansible.builtin.copy: @@ -30,12 +30,6 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} -- name: Sync the repositories - community.general.portage: - sync: 'yes' - verbose: true - when: overlay_config_files is changed - - name: Find all files and directories in the etc/portage directory of the overlay ansible.builtin.find: file_type: any From d38d6dd369944f4acb86c38672f44d33740a4249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:36:03 +0200 Subject: [PATCH 07/36] flush handlers --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 16f75e0b..693306cb 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -30,6 +30,9 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} +- name: Flush handlers to make sure that overlays are synced + meta: flush_handlers + - name: Find all files and directories in the etc/portage directory of the overlay ansible.builtin.find: file_type: any From 7cf15eff3c798ea39b03eb1a06f4bd0b4afc6ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:48:46 +0200 Subject: [PATCH 08/36] remove blank line --- ansible/playbooks/roles/compatibility_layer/handlers/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml index 195b1912..078eb915 100644 --- a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml @@ -9,4 +9,3 @@ community.general.portage: sync: 'yes' verbose: true - From 3d15aa0d4f49c8280164752e531db35f7306a94a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:49:00 +0200 Subject: [PATCH 09/36] use ansible.builtin.meta --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 693306cb..47a8c4d4 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -31,7 +31,7 @@ }} - name: Flush handlers to make sure that overlays are synced - meta: flush_handlers + ansible.builtin.meta: flush_handlers - name: Find all files and directories in the etc/portage directory of the overlay ansible.builtin.find: From 2eb4c2af10ed17ca0a464ac13557230ff7dcea97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:56:58 +0200 Subject: [PATCH 10/36] add community.general dependency --- .github/workflows/ansible-lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index 68ba88ac..9b765099 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -21,3 +21,4 @@ jobs: # demote var-naming[no-role-prefix] to warnings, as we only have a single role, # and prefixing all variables in that role with the role name is really ugly args: "--warn-list var-naming[no-role-prefix]" + required_collections: 'community.general' From 5647d518b71db87aa0babb2244ab502a13a9a26b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 20:59:51 +0200 Subject: [PATCH 11/36] galaxy requirements file --- ansible/playbooks/galaxy-requirements.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 ansible/playbooks/galaxy-requirements.yml diff --git a/ansible/playbooks/galaxy-requirements.yml b/ansible/playbooks/galaxy-requirements.yml new file mode 100644 index 00000000..06b90316 --- /dev/null +++ b/ansible/playbooks/galaxy-requirements.yml @@ -0,0 +1,8 @@ +# +# Install roles and collections from the default Ansible Galaxy server. +# +--- +collections: + - name: community.general + version: '>=10.7.3' +... From 0d18772c2d042470312a56b56a96cd26a2e1c55b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 21:00:26 +0200 Subject: [PATCH 12/36] point to requirements file --- .github/workflows/ansible-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index 9b765099..b1c36fe7 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -21,4 +21,4 @@ jobs: # demote var-naming[no-role-prefix] to warnings, as we only have a single role, # and prefixing all variables in that role with the role name is really ugly args: "--warn-list var-naming[no-role-prefix]" - required_collections: 'community.general' + requirements_file: "galaxy-requirements.yml" From 5e1db9b46714d55df2ff869f3a4b649bd3b823fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 21:01:23 +0200 Subject: [PATCH 13/36] correct path to requirements file --- .github/workflows/ansible-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index b1c36fe7..c64abe87 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -21,4 +21,4 @@ jobs: # demote var-naming[no-role-prefix] to warnings, as we only have a single role, # and prefixing all variables in that role with the role name is really ugly args: "--warn-list var-naming[no-role-prefix]" - requirements_file: "galaxy-requirements.yml" + requirements_file: "ansible/playbooks/galaxy-requirements.yml" From 5b415fa3d03fafd8579df0702aff5a2a0b3ba90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 21:02:33 +0200 Subject: [PATCH 14/36] move requirements file --- ansible/playbooks/galaxy-requirements.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 ansible/playbooks/galaxy-requirements.yml diff --git a/ansible/playbooks/galaxy-requirements.yml b/ansible/playbooks/galaxy-requirements.yml deleted file mode 100644 index 06b90316..00000000 --- a/ansible/playbooks/galaxy-requirements.yml +++ /dev/null @@ -1,8 +0,0 @@ -# -# Install roles and collections from the default Ansible Galaxy server. -# ---- -collections: - - name: community.general - version: '>=10.7.3' -... From bf193be93895b32e6d1c17da6749cef33a766f98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 21:02:38 +0200 Subject: [PATCH 15/36] move requirements file --- ansible/galaxy-requirements.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 ansible/galaxy-requirements.yml diff --git a/ansible/galaxy-requirements.yml b/ansible/galaxy-requirements.yml new file mode 100644 index 00000000..06b90316 --- /dev/null +++ b/ansible/galaxy-requirements.yml @@ -0,0 +1,8 @@ +# +# Install roles and collections from the default Ansible Galaxy server. +# +--- +collections: + - name: community.general + version: '>=10.7.3' +... From 419e3ce927f3a6b660da6265f6c0c04666c9f3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Mon, 15 Sep 2025 21:02:59 +0200 Subject: [PATCH 16/36] move requirements file --- .github/workflows/ansible-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index c64abe87..94679101 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -21,4 +21,4 @@ jobs: # demote var-naming[no-role-prefix] to warnings, as we only have a single role, # and prefixing all variables in that role with the role name is really ugly args: "--warn-list var-naming[no-role-prefix]" - requirements_file: "ansible/playbooks/galaxy-requirements.yml" + requirements_file: "ansible/galaxy-requirements.yml" From f6e57ebdf6257a26bd5f3592a39ed13ecb733bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 16 Dec 2025 14:21:50 +0100 Subject: [PATCH 17/36] adjust trusted dirs test --- test/compat_layer.py | 53 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/test/compat_layer.py b/test/compat_layer.py index fb494fc3..746a5f1d 100644 --- a/test/compat_layer.py +++ b/test/compat_layer.py @@ -219,15 +219,20 @@ def __init__(self): self.descr = 'Verify that the env file for sys-libs/glibc was created and is picked up by emerge.' self.command = 'equery has --package glibc EXTRA_EMAKE' - trusted_dir = os.path.join( - self.eessi_repo_dir, - 'host_injections', - self.eessi_version, - 'compat', - self.eessi_os, - self.eessi_arch, - 'lib' - ) + # in 2023.06 we had a single trusted directory in host_injections, + # in 2025.06 we introduced three subdirectories (override, nvidia, amd) in the lib dir of the compat layer itself. + if self.eessi_version == '2023.06': + trusted_dirs = os.path.join( + self.eessi_repo_dir, + 'host_injections', + self.eessi_version, + 'compat', + self.eessi_os, + self.eessi_arch, + 'lib' + ) + else: + trusted_dirs = [os.path.join(self.compat_dir, 'lib', subdir) for subdir in ['override', 'nvidia', 'amd']] self.sanity_patterns = sn.assert_found( f'user-defined-trusted-dirs={trusted_dir}', @@ -242,26 +247,20 @@ def __init__(self): self.descr = 'Verify that glibc was compiled with the custom user-defined trusted dirs.' self.command = 'ld.so --help' - libdir = os.path.join( - self.eessi_repo_dir, - 'host_injections', - self.eessi_version, - 'compat', - self.eessi_os, - self.eessi_arch, - 'lib' - ) - - # in 2023.06 we had a single trusted directory, - # in 2025.06 we introduced three subdirectories (override, nvidia, amd). + # in 2023.06 we had a single trusted directory in host_injections, + # in 2025.06 we introduced three subdirectories (override, nvidia, amd) in the lib dir of the compat layer itself. if self.eessi_version == '2023.06': - trusted_dirs = [libdir] + trusted_dirs = os.path.join( + self.eessi_repo_dir, + 'host_injections', + self.eessi_version, + 'compat', + self.eessi_os, + self.eessi_arch, + 'lib' + ) else: - trusted_dirs = [ - os.path.join(libdir, 'override'), - os.path.join(libdir, 'nvidia'), - os.path.join(libdir, 'amd'), - ] + trusted_dirs = [os.path.join(self.compat_dir, 'lib', subdir) for subdir in ['override', 'nvidia', 'amd']] # ld.so --help prints the trusted directories as: # /path/to/dir (system search path) From c4d424f43cb3315be154ea480969dbb86bb4e503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 29 Jan 2026 13:16:18 +0100 Subject: [PATCH 18/36] handle new compat layers and updates --- bot/build.sh | 58 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index d59e1505..0bca52ae 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -85,27 +85,47 @@ job_repo=$(cfg_get_value "repository" "repo_name") eessi_repo=${job_repo:-software.eessi.io} tar_topdir=/cvmfs/${eessi_repo}/versions -if [ "${eessi_arch}" != "${host_arch}" ]; then - echo "Requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!" +# Check the architecture, which has to match the architecture of the build host +if [[ "${eessi_arch}" != "${host_arch}" ]]; then + echo "bot/build.sh: requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!" exit 1 fi -# option -k is used for retaining ${eessi_tmp} -# store output in local file such that the temporary directory ${STORAGE}/eessi.XXXXXXXXXX -# can be determined +# Read and check the build type (new or update, case-insensitive) +if [[ ! -f bot/build_type ]]; then + echo 'bot/build.sh: cannot determine build type, please add a file "bot/build_type" containing either "new" or "update".' + exit 1 +fi +build_type=$(cat bot/build_type | tr '[:upper:]' '[:lower:]') +if [[ "${build_type}" != "new" && "${build_type}" != "update" ]]; then + echo 'bot/build.sh: invalid build type! It has to be "new" or "update".' + exit 1 +fi +echo "bot/build.sh: requested build type '${build_type}'" + +# Set up the script arguments and run the installation script script_out="install_stdout.log" -./install_compatibility_layer.sh -a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k --verbose 2>&1 | tee -a ${script_out} - -# TODO handle errors (no outfile, no tmp directory found) -eessi_tmp=$(cat ${script_out} | grep 'To resume work add' | cut -f 2 -d \' | cut -f 2 -d ' ') -eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) -# create tarball -> should go into a separate script when this is supported by the bot -target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz -if [ -d ${eessi_tmp}${tar_topdir}/${eessi_version} ]; then - echo ">> Creating tarball ${target_tgz} from ${eessi_tmp}${tar_topdir}..." - tar cfvz ${target_tgz} -C ${eessi_tmp}${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} - echo ${target_tgz} created! -else - echo "Directory ${eessi_tmp}${tar_topdir}/${eessi_version} was not created, not creating tarball." - exit 1 +script_args=(-a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k) +# for (very!) verbose output, uncomment: +#script_args+=(--verbose) +if [[ "${build_type}" == "update" ]]; then + script_args+=(-u) +fi +./install_compatibility_layer.sh ${script_args[@]} 2>&1 | tee -a ${script_out} + +# Create a tarball of the new or updated build +# TODO handle errors (no outfile, no tmp directory found, etc) +eessi_tmp=$(grep '^To resume work add' ${script_out} | cut -f 2 -d \' | cut -f 2 -d ' ' | tail -n 1) +echo "bot/build.sh: creating compatibility layer tarball..." +if [[ "${build_type}" == "update" ]]; then + # Resume the build container session and tar the entire /cvmfs/$repo tree + container_tmp=$(grep -oP '(?<=^Using ).*(?= as tmp directory)' ${script_out} | tail -n 1) + eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions) + target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz + ${eessi_tmp}/software-layer-scripts/eessi_container.sh --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} +elif [[ "${build_type}" == "new" ]]; then + # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo + eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) + target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz + tar cfvz ${target_tgz} -C ${eessi_tmp}${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} fi From f493340bb08fa8b2e62e836cdd3f9d11a562249f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 29 Jan 2026 13:16:35 +0100 Subject: [PATCH 19/36] file that controls whether this is a new or updated compat layer --- bot/build_type | 1 + 1 file changed, 1 insertion(+) create mode 100644 bot/build_type diff --git a/bot/build_type b/bot/build_type new file mode 100644 index 00000000..4ea5e4dd --- /dev/null +++ b/bot/build_type @@ -0,0 +1 @@ +update From e97e3c50999fe0e922d765f78fab572c8a15c6b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 29 Jan 2026 13:16:55 +0100 Subject: [PATCH 20/36] use eessi_container.sh, allow for both new and updated compat layers --- install_compatibility_layer.sh | 73 ++++++++++++++++------------------ 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh index ac70271d..833713ee 100755 --- a/install_compatibility_layer.sh +++ b/install_compatibility_layer.sh @@ -5,17 +5,18 @@ # ARCH= -CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian11 +CONTAINER=docker://ghcr.io/eessi/build-node-compat-layer:debian-12 REPOSITORY="software.eessi.io" RESUME= RETAIN_TMP=0 STORAGE= +UPDATE=0 VERSION= VERBOSE= # Debian 11 does not support RISC-V, so we use a Debian 13 container instead. if [[ $(uname -m) = "riscv64" ]]; then - CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian-13 + CONTAINER=docker://ghcr.io/eessi/build-node-compat-layer:debian-13 fi display_help() { @@ -46,6 +47,11 @@ display_help() { echo " -t | --resume TMPDIR" echo " tmp directory to resume from [default: None]" echo "" + echo " -u | --update" + echo " update an existing compatibility layer by" + echo " doing a fuse mount of the given repository" + echo " [default: not set]" + echo "" echo " -v | --version VERSION" echo " override the EESSI stack version set in Ansible's" echo " defaults/main.yml file [default: None]" @@ -87,6 +93,10 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; + -u|--update) + UPDATE=1 + shift + ;; -v|--version) VERSION="$2" shift 2 @@ -116,9 +126,6 @@ if [ ! -f "${SCRIPT_DIR}/ansible/playbooks/install.yml" ]; then exit 1 fi -# source utils.sh (for get_container_runtime and check_exit_code) -source ${SCRIPT_DIR}/scripts/utils.sh - # Check if the target architecture is set to the architecture of the current host, # as that's the only thing that's currently supported by this script HOST_ARCH=$(uname -m) @@ -129,7 +136,6 @@ fi if [[ -z ${ARCH} ]]; then ARCH=${HOST_ARCH} fi -echo "A compatibility layer for architecture ${ARCH} will be built." # Make a temporary directory on the host for storing the installation and some temporary files if [[ ! -z ${RESUME} ]] && [[ -d ${RESUME} ]]; then @@ -143,40 +149,12 @@ else fi echo "Using $EESSI_TMPDIR as temporary storage..." -# Create temporary directories -mkdir -p ${EESSI_TMPDIR}/cvmfs -mkdir -p ${EESSI_TMPDIR}/home -mkdir -p ${EESSI_TMPDIR}/tmp - -RUNTIME=$(get_container_runtime) -exit_code=$? -[[ ${VERBOSE} == '-vvv' ]] && echo "RUNTIME='${RUNTIME}'" -check_exit_code ${exit_code} "using runtime ${RUNTIME}" "oh no, neither apptainer nor singularity available" - -# Set up paths and mount points for Apptainer -if [[ -z ${APPTAINER_CACHEDIR} ]]; then - export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache - [[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_CACHEDIR='${APPTAINER_CACHEDIR}'" -fi -export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" -export APPTAINER_BIND="${APPTAINER_BIND},${EESSI_TMPDIR}/tmp:/tmp" -[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_BIND='${APPTAINER_BIND}'" -export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}" -[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_HOME='${APPTAINER_HOME}'" - -# also define SINGULARITY_* env vars -if [[ -z ${SINGULARITY_CACHEDIR} ]]; then - export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache - [[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" -fi -export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" -export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/tmp:/tmp" -[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" -export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" -[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'" +# Clone the EESSI/software-layer-scripts repository +git clone https://github.com/EESSI/software-layer-scripts ${EESSI_TMPDIR}/software-layer-scripts +#cp ../eessi_container.sh ${EESSI_TMPDIR}/software-layer-scripts/eessi_container.sh # Construct the Ansible playbook command -ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=$(uname -m)" +ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=${ARCH}" if [[ ! -z ${VERSION} ]]; then ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} -e eessi_version=${VERSION}" fi @@ -187,12 +165,29 @@ if [[ ! -z ${VERBOSE} ]]; then ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} ${VERBOSE}" fi ANSIBLE_COMMAND="ansible-playbook ${ANSIBLE_OPTIONS} /compatibility-layer/ansible/playbooks/install.yml" + +# Set the options for the EESSI container script +CONTAINER_OPTIONS="-c ${CONTAINER} -g ${EESSI_TMPDIR}" +if [[ $UPDATE -eq 0 ]]; then + # For a new compatibility layer, we bind mount an empty host directory as /cvmfs. + # This is a lot faster than (unnecessarily) using an overlay on top of a fuse-mounted /cvmfs. + mkdir "${EESSI_TMPDIR}/cvmfs" + CONTAINER_OPTIONS="${CONTAINER_OPTIONS} -r none -b ${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" +else + # To update an existing compatibility layer, we do have to use an overlay. + CONTAINER_OPTIONS="${CONTAINER_OPTIONS} --access rw -r ${REPOSITORY} -b ${SCRIPT_DIR}:/compatibility-layer" +fi + # Finally, run Ansible inside the container to do the actual installation echo "Executing ${ANSIBLE_COMMAND} in ${CONTAINER}, this will take a while..." -${RUNTIME} shell ${CONTAINER} < Date: Thu, 29 Jan 2026 13:47:20 +0100 Subject: [PATCH 21/36] reuse container image for tarball step --- bot/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0bca52ae..09a0e002 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -120,9 +120,10 @@ echo "bot/build.sh: creating compatibility layer tarball..." if [[ "${build_type}" == "update" ]]; then # Resume the build container session and tar the entire /cvmfs/$repo tree container_tmp=$(grep -oP '(?<=^Using ).*(?= as tmp directory)' ${script_out} | tail -n 1) + container_image=$(ls -1t ${container_tmp}/*.sif | head -n 1) eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions) target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz - ${eessi_tmp}/software-layer-scripts/eessi_container.sh --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} + ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_img} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} elif [[ "${build_type}" == "new" ]]; then # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) From 62926fe6e45622d5ed0ed8458102c72a33b3f3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 29 Jan 2026 13:54:40 +0100 Subject: [PATCH 22/36] only do reprod step for new builds --- ansible/playbooks/roles/compatibility_layer/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml index 7a45863a..07ef41b3 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml @@ -51,3 +51,4 @@ ansible.builtin.include_tasks: reprod.yml tags: - reprod + when: not startprefix.stat.exists From fbffc5a4de1750c2339999364dae79c4b729124f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 29 Jan 2026 14:25:30 +0100 Subject: [PATCH 23/36] fix variable name --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 09a0e002..f933289c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -123,7 +123,7 @@ if [[ "${build_type}" == "update" ]]; then container_image=$(ls -1t ${container_tmp}/*.sif | head -n 1) eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions) target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz - ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_img} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} + ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_image} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} elif [[ "${build_type}" == "new" ]]; then # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) From 6ced9cbec7ed43b7d8fae6ae86515025c4da4c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 29 Jan 2026 15:18:46 +0100 Subject: [PATCH 24/36] run reprod for updates as well, use timestamped dir, don't copy bootstrap script for updates --- .../roles/compatibility_layer/tasks/main.yml | 1 - .../roles/compatibility_layer/tasks/reprod.yml | 11 +++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml index 07ef41b3..7a45863a 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml @@ -51,4 +51,3 @@ ansible.builtin.include_tasks: reprod.yml tags: - reprod - when: not startprefix.stat.exists diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml index 36931898..13ca371d 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml @@ -1,9 +1,15 @@ # Store some information and scripts that were used for this installation. --- -- name: Make a subdirectory for storing build information +- name: Determine timestamped reprod directory for storing information about this build + ansible.builtin.set_fact: + reprod_dir: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ '%Y%m%d_%H%M%SUTC' | strftime(ansible_facts.date_time.epoch | int, utc=True) }}" + tags: + - reprod + +- name: Make a timestamped subdirectory for storing build information ansible.builtin.file: - path: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}" + path: "{{ reprod_dir }}" state: directory mode: '0755' tags: @@ -16,6 +22,7 @@ mode: '0644' tags: - reprod + when: not startprefix.stat.exists - name: Get list of installed packages ansible.builtin.command: "qlist -IRv" From a82e84c3a249fd9e2a303efcdf04c3ef1c9d86a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 29 Jan 2026 15:55:25 +0100 Subject: [PATCH 25/36] use reprod_dir everywhere --- .../playbooks/roles/compatibility_layer/tasks/reprod.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml index 13ca371d..335e8c94 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml @@ -18,7 +18,7 @@ - name: Copy the used bootstrap script ansible.builtin.copy: src: "{{ prefix_use_builtin_bootstrap | ternary('/usr/local/bin/bootstrap-prefix.sh', prefix_custom_bootstrap_script.remote) }}" - dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/bootstrap-prefix.sh" + dest: "{{ reprod_dir }}/bootstrap-prefix.sh" mode: '0644' tags: - reprod @@ -34,7 +34,7 @@ - name: Dump list of installed packages to a file ansible.builtin.copy: content: "{{ qlist.stdout }}" - dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ prefix_packages_file }}" + dest: "{{ reprod_dir }}/{{ prefix_packages_file }}" mode: '0644' tags: - reprod @@ -42,7 +42,7 @@ - name: Store other metadata of build in a json file ansible.builtin.copy: content: "{{ metadata | to_nice_json }}" - dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ prefix_metadata_json }}" + dest: "{{ reprod_dir }}/{{ prefix_metadata_json }}" mode: '0644' vars: metadata: From 46594b6e78b17d2e045da5ff308fca60a8b3321b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 30 Jan 2026 15:57:32 +0100 Subject: [PATCH 26/36] do an explicit sync of the overlays --- .../roles/compatibility_layer/tasks/add_overlay.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 47a8c4d4..f085523c 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -17,7 +17,6 @@ dest: "{{ gentoo_prefix_path }}/etc/portage/repos.conf/{{ item.name }}.conf" mode: "0644" loop: "{{ custom_overlays }}" - notify: Sync overlays - name: Make configuration file with overlays that can override eclasses ansible.builtin.copy: @@ -30,8 +29,10 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} -- name: Flush handlers to make sure that overlays are synced - ansible.builtin.meta: flush_handlers +- name: Sync the overlays to make sure that they are up to date + community.general.portage: + sync: 'yes' + verbose: true - name: Find all files and directories in the etc/portage directory of the overlay ansible.builtin.find: From d6179410d97710f5038043181bab44e51de4f497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 30 Jan 2026 16:38:07 +0100 Subject: [PATCH 27/36] update portage files --- .../tasks/prefix_configuration.yml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml index c69f5ee2..d8d0f1a5 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml @@ -2,6 +2,27 @@ # for instance locale support. --- +- name: "Update package.mask file" + ansible.builtin.copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.mask" + content: "{{ prefix_mask_packages }}" + mode: "0644" + when: prefix_mask_packages is defined and prefix_mask_packages | length > 0 + +- name: "Update package.unmask file" + ansible.builtin.copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.unmask" + content: "{{ prefix_unmask_packages }}" + mode: "0644" + when: prefix_unmask_packages is defined and prefix_unmask_packages | length > 0 + +- name: "Update package.use file" + ansible.builtin.copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.use" + content: "{{ prefix_bootstrap_use_flags }}" + mode: "0644" + when: prefix_bootstrap_use_flags is defined and prefix_bootstrap_use_flags | length > 0 + - name: Add locales to configuration file ansible.builtin.lineinfile: path: "{{ gentoo_prefix_path }}/etc/locale.gen" From b8e83b5fd24ee3f6dd4b59f58b9b6580156c39b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 30 Jan 2026 17:28:47 +0100 Subject: [PATCH 28/36] tarball step needs access rw, otherwise the overlay-upper is not used --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index f933289c..02125af8 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -123,7 +123,7 @@ if [[ "${build_type}" == "update" ]]; then container_image=$(ls -1t ${container_tmp}/*.sif | head -n 1) eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions) target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz - ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_image} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} + ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_image} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job --access rw -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} elif [[ "${build_type}" == "new" ]]; then # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) From 14040a3da5df1e5741c3789ee59455ec09dcd972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:08:50 +0100 Subject: [PATCH 29/36] add comments about handlers --- ansible/playbooks/roles/compatibility_layer/handlers/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml index 078eb915..447d7dcf 100644 --- a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml @@ -1,10 +1,14 @@ # Handlers for this role --- +# Handler for (re)generating locales - name: Generate locales ansible.builtin.command: locale-gen changed_when: true +# Handler for syncing the Gentoo overlays; +# note that it's currently not used, as we do an explicit sync anyway in the playbook, +# but it could still be useful - name: Sync overlays community.general.portage: sync: 'yes' From d62e7b1eaf6e8a32ea29f32b53d1ded5d4deb5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:12:54 +0100 Subject: [PATCH 30/36] add comment about syncing overlays --- .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index f085523c..db3c6471 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -29,6 +29,8 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} +# We do an explicit sync here (instead of using a handler), +# so we can be sure that the overlays are up-to-date. - name: Sync the overlays to make sure that they are up to date community.general.portage: sync: 'yes' From 0e260854e4648e9719b7786b42c056fc6a1d099c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:14:23 +0100 Subject: [PATCH 31/36] add todo about code duplication --- .../roles/compatibility_layer/tasks/prefix_configuration.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml index d8d0f1a5..3453b927 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml @@ -2,6 +2,8 @@ # for instance locale support. --- +# To avoid code duplication, the tasks for updating these portage files could be moved to a separate file, +# which is then included by both prefix_configuration.yml and install_prefix.yml. - name: "Update package.mask file" ansible.builtin.copy: dest: "{{ gentoo_prefix_path }}/etc/portage/package.mask" From e38a2e4975a3ec8cd4e0a156453c0d227f9c69a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:16:04 +0100 Subject: [PATCH 32/36] comment about reprod dirs --- ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml index 335e8c94..f26e4f59 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml @@ -1,4 +1,7 @@ -# Store some information and scripts that were used for this installation. +# Store some information and scripts that were used for this installation +# in $EPREFIX/reprod/YYYYMMDD_hhmmssUTC. +# The timestamped subdir ensures that updates of the compatibility layer +# don't overwrite these files; instead they end up in their own subdir. --- - name: Determine timestamped reprod directory for storing information about this build From 14c2e1b9830e00fd7fa2e9f57b5ed85253c5b1cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:18:14 +0100 Subject: [PATCH 33/36] comment about bootstrap script --- ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml index f26e4f59..4064592d 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml @@ -25,6 +25,7 @@ mode: '0644' tags: - reprod + # Skip this step for updates, as these don't use the bootstrap script anyway when: not startprefix.stat.exists - name: Get list of installed packages From 4f315d5b65995277c26648b25bea117c4507d26c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:19:21 +0100 Subject: [PATCH 34/36] add comment about access rw in tarball step --- bot/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bot/build.sh b/bot/build.sh index 02125af8..629fad87 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -123,6 +123,7 @@ if [[ "${build_type}" == "update" ]]; then container_image=$(ls -1t ${container_tmp}/*.sif | head -n 1) eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions) target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz + # note: "--access rw" is important, as we need to (re)use the overlay! ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_image} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job --access rw -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} elif [[ "${build_type}" == "new" ]]; then # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo From 0c9a9141f65ecbcb30a459e4ececf7b79425d7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:21:22 +0100 Subject: [PATCH 35/36] add comment about determining the EESSI version --- bot/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 629fad87..bf1bbcdc 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -121,12 +121,14 @@ if [[ "${build_type}" == "update" ]]; then # Resume the build container session and tar the entire /cvmfs/$repo tree container_tmp=$(grep -oP '(?<=^Using ).*(?= as tmp directory)' ${script_out} | tail -n 1) container_image=$(ls -1t ${container_tmp}/*.sif | head -n 1) + # Find the EESSI version by checking which version was changed by the installation script (only one version could have been changed) eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions) target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz # note: "--access rw" is important, as we need to (re)use the overlay! ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_image} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job --access rw -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} elif [[ "${build_type}" == "new" ]]; then # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo + # Find the EESSI version by checking which version was created by the installation script in the host directory eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz tar cfvz ${target_tgz} -C ${eessi_tmp}${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} From d6872878caeacff1e0c75be608f5c422673329f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Tue, 3 Feb 2026 13:22:43 +0100 Subject: [PATCH 36/36] modify comment about prefix/reframe env vars --- install_compatibility_layer.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh index 833713ee..026b25fb 100755 --- a/install_compatibility_layer.sh +++ b/install_compatibility_layer.sh @@ -181,7 +181,8 @@ fi # Finally, run Ansible inside the container to do the actual installation echo "Executing ${ANSIBLE_COMMAND} in ${CONTAINER}, this will take a while..." ${EESSI_TMPDIR}/software-layer-scripts/eessi_container.sh ${CONTAINER_OPTIONS} <