Bump version to 2021.12, use new snapshot and new bootstrap script + mask glibc >= 2.34

ansible/playbooks/README.md

@@ -42,8 +42,10 @@ Before running the playbook, make sure the following settings are correct, and o
 | prefix_required_space | Minimal amount of disk space that is required for the Gentoo Prefix bootstrap |
 | prefix_snapshot_url | Directory (served over http(s)) containing snapshot files |
 | prefix_snapshot_version | Date (`YYYYMMDD`) of the Portage snapshot file for the Prefix installation |
-| prefix_python_targets | String consisting of [Gentoo Python targets](https://wiki.gentoo.org/wiki/Project:Python/PYTHON_TARGETS) Python targets used for the Prefix installation |
 | prefix_user_defined_trusted_dirs | List of paths to the user defined trusted dirs for glibc |
+| prefix_mask_packages | Contents of a [package.mask file](https://wiki.gentoo.org/wiki//etc/portage/package.mask) that should be used during the bootstrap |
+| prefix_use_builtin_bootstrap | Use the container's built-in bootstrap script? |
+| prefix_custom_bootstrap_script | Dictionary with the `local` source and `remote` destination of the bootstrap script |
 | prefix_singularity_command | Singularity command for launching the container with the bootstrap script |
 | prefix_source | Singularity container path used for the Prefix installtion |
 | prefix_source_options | Arguments to be passed to the Prefix bootstrap script |
@@ -75,3 +77,7 @@ ip-or-hostname-of-your-stratum0 eessi_host_arch=x86_64 eessi_host_os=linux
 
 The `eessi_host_arch` corresponds to the architecture of the machine that executes the playbook and for which this compatibility layer has to be built, e.g. `x86_64`, `aarch64`, or `ppc64le`.
 Similarly, `eessi_host_os` should refer to the operating system of the machine, and should be set to either `linux` or `macos`.
+
+Note that, by default, the bootstrap script from the root directory of this git repository will be used, unless you set `prefix_use_builtin_bootstrap` to `yes` 
+(in which case the bootstrap script included in the Prefix build container will be used).
+A different bootstrap script can be used by changing `prefix_custom_bootstrap_script.local` to a local path.

ansible/playbooks/roles/compatibility_layer/defaults/main.yml

@@ -1,6 +1,6 @@
 # Defaults file for the compatibility layer role.
 ---
-eessi_version: "2021.06"
+eessi_version: "2021.12"
 
 custom_overlays:
   - name: eessi
@@ -18,17 +18,30 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/{{ eessi_version }}/compat/{{
 # How to build the prefix
 prefix_required_space: 15 GB
 prefix_snapshot_url: https://eessi-gentoo-snapshot.s3-eu-west-1.amazonaws.com
-prefix_snapshot_version: 20210607
+prefix_snapshot_version: 20211120
 prefix_user_defined_trusted_dirs:
   - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib"
-prefix_singularity_command: "singularity run -B {{ gentoo_prefix_path }}:{{ gentoo_prefix_path }}"
+prefix_mask_packages: |
+  # avoid glibc 2.34, as it's causing issues with the bootstrap, and it's not compatible with CUDA 11.
+  # see https://github.com/EESSI/compatibility-layer/issues/137 + https://bugs.gentoo.org/824482
+  >=sys-libs/glibc-2.34
+  # avoid libgcrypt 1.9.4 due to compiler errros on ppc64le,
+  # see https://github.com/EESSI/compatibility-layer/issues/134 + https://bugs.gentoo.org/825722
+  =dev-libs/libgcrypt-1.9.4
+prefix_use_builtin_bootstrap: no
+prefix_custom_bootstrap_script:
+  local: "{{ playbook_dir }}/../../bootstrap-prefix.sh"
+  remote: /tmp/bootstrap-prefix.sh
+prefix_singularity_command: "singularity exec -B {{ gentoo_prefix_path }}:{{ gentoo_prefix_path }}"
 prefix_source: "docker://ghcr.io/eessi/bootstrap-prefix:centos8"
 prefix_source_options: "{{ gentoo_prefix_path }} noninteractive"
 prefix_install: >-
     SINGULARITYENV_USE_CPU_CORES={{ ansible_processor_vcpus }}
     SINGULARITYENV_CUSTOM_SNAPSHOT_URL="{{ prefix_snapshot_url }}"
     SINGULARITYENV_CUSTOM_SNAPSHOT_VERSION="{{ prefix_snapshot_version }}"
-    {{ prefix_singularity_command }} {{ prefix_source }} {{ prefix_source_options }}
+    {{ prefix_singularity_command }} {{ prefix_source }}
+    {{ prefix_use_builtin_bootstrap | ternary('/usr/local/bin/bootstrap-prefix.sh', prefix_custom_bootstrap_script.remote) }}
+    {{ prefix_source_options }}
 
 # Logging
 eessi_log_dir: "/tmp/eessi-logs"

ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml

@@ -61,6 +61,33 @@
   tags:
     - build_prefix
 
+- name: "Copy custom bootstrap script"
+  copy:
+    src: "{{ prefix_custom_bootstrap_script.local }}"
+    dest: "{{ prefix_custom_bootstrap_script.remote }}"
+    mode: 0755
+  when: not prefix_use_builtin_bootstrap
+  tags:
+    - build_prefix
+
+- name: "Create $EPREFIX/etc/portage directory"
+  file:
+    path: "{{ gentoo_prefix_path }}/etc/portage"
+    state: directory
+    mode: 0755
+  when: prefix_mask_packages is defined and prefix_mask_packages | length > 0
+  tags:
+    - build_prefix
+
+- name: "Mask packages for the bootstrap"
+  copy:
+    dest: "{{ gentoo_prefix_path }}/etc/portage/package.mask"
+    content: "{{ prefix_mask_packages }}"
+    mode: 0644
+  when: prefix_mask_packages is defined and prefix_mask_packages | length > 0
+  tags:
+    - build_prefix
+
 - name: "Give {{ username_on_host.stdout }} recursive user and group ownership of {{ gentoo_prefix_path }}"
   file:
     dest: "{{ gentoo_prefix_path }}"
@@ -81,8 +108,12 @@
   stat:
     path: "{{ gentoo_prefix_path }}/startprefix"
   register: startprefix
+  tags:
+    - build_prefix
 
 - name: "Fail if startprefix script has not been created"
   fail:
     msg: "The resulting Gentoo Prefix installation does not have a startprefix script. Something went wrong!"
   when: not startprefix.stat.exists
+  tags:
+    - build_prefix

ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml

@@ -1,15 +1,12 @@
 # Make sure that glibc is always compiled with a user-defined-trusted-dirs option
 ---
-- name: Find all strings in libc library
-  command: "strings {{ gentoo_prefix_path }}/usr/lib64/libc.a"
-  register: libc_strings
+- name: Check the EXTRA_EMAKE flags of glibc
+  command: "equery has --package glibc EXTRA_EMAKE"
+  changed_when: false
+  ignore_errors: yes
+  register: glibc_extra_emake
   when: eessi_host_os == "linux"
 
-- name: Find user defined trusted dirs in libc strings output
-  set_fact: match='{{ libc_strings.stdout | regex_search("\n" + item + "/?\n") | default('', True) | string | length>0 }}'
-  with_items: "{{ prefix_user_defined_trusted_dirs }}"
-  register: trusted_dirs_in_libc
-
 - name: (Re)install glibc with the user-defined-trusted-dirs option
   portage:
     package: sys-libs/glibc
@@ -20,7 +17,7 @@
     EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
   when:
     - eessi_host_os == "linux"
-    - trusted_dirs_in_libc.results | selectattr('ansible_facts.match', 'equalto', False) | list | length>0
+    - glibc_extra_emake.stdout != "user-defined-trusted-dirs=" + ":".join(prefix_user_defined_trusted_dirs)
 
 - name: Create portage env directory
   file: