From b8c1b8e5e8e8fa3af447a435558b2c4ea5c28ddb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 11:24:46 +0200
Subject: [PATCH 01/25] added helper scripts and improved bot/build.sh

- scripts/utils.sh is borrowed from software-layer. it includes various functions
  to be used with other scripts in this repo.
- scripts/cfg_files.sh is borrowed from software-layer. it provides functions to
  read a file that contains information about a job.
- changes to bot/build.sh:
  - added header, author, license
  - added assumptions
  - added set -e
  - source the above helper scripts
  - read job.cfg file
  - set core settings if needed: HTTP_PROXY, HTTPS_PROXY, LOCAL_TMP, LOAD_MODULES
  - expand LOCAL_TMP into STORAGE
  - load modules
  - obtain cpu_target_arch from job.cfg setting
  - bump eessi_version to 2023.04
  - calling install_compatibility_layer.sh: remove fixed local container, add
    storage option
---
 bot/build.sh         |  85 +++++++++++++++++++++-
 scripts/cfg_files.sh | 167 +++++++++++++++++++++++++++++++++++++++++++
 scripts/utils.sh     | 117 ++++++++++++++++++++++++++++++
 3 files changed, 366 insertions(+), 3 deletions(-)
 create mode 100644 scripts/cfg_files.sh
 create mode 100644 scripts/utils.sh

diff --git a/bot/build.sh b/bot/build.sh
index 4833ac85..6011cd87 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -1,10 +1,88 @@
 #!/bin/bash
+#
+# script to build the EESSI compatibility layer. Intended use is that it is called
+# by a (batch) job running on a compute node.
+#
+# This script is part of the EESSI compatibility layer, see
+# https://github.com/EESSI/compatibility-layer.git
+#
+# author: Bob Droege (@bedroge)
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
 
-cpu_target_arch=$(echo ${CPU_TARGET} | cut -d/ -f1)
+# ASSUMPTIONs:
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files referenced in job.cfg
+
+# stop as soon as something fails
+set -e
+
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
+
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
+echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'"
+
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
+echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
+
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
+echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
+
+echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+# replace any env variable in ${LOCAL_TMP} with its
+#   current value (e.g., a value that is local to the job)
+STORAGE=$(envsubst <<< ${LOCAL_TMP})
+echo "'${STORAGE}'"
+
+# make sure ${STORAGE} exists
+mkdir -p ${STORAGE}
+
+# obtain list of modules to be loaded
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
+echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
+
+# load modules if LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/build.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/build.sh: no modules to be loaded"
+fi
+
+#####################################################
+# cpu_target_arch=$(echo ${CPU_TARGET} | cut -d/ -f1)
+cpu_target_arch=$(cfg_get_value "architecture" "software_subdir" | cut -d/ -f1)
 host_arch=$(uname -m)
 eessi_arch=${cpu_target_arch:-${host_arch}}
 eessi_os=linux
-eessi_version=2023.02
+eessi_version=2023.04
 eessi_repo=pilot.eessi-hpc.org
 tar_topdir=/cvmfs/${eessi_repo}/versions
 
@@ -13,7 +91,8 @@ if [ "${eessi_arch}" != "${host_arch}" ]; then
   exit 1
 fi
 
-./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -c ~/compat.sif
+#./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -c ~/compat.sif
+./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -g ${STORAGE}
 
 # create tarball -> should go into a separate script when this is supported by the bot
 target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz
diff --git a/scripts/cfg_files.sh b/scripts/cfg_files.sh
new file mode 100644
index 00000000..57ea2f7c
--- /dev/null
+++ b/scripts/cfg_files.sh
@@ -0,0 +1,167 @@
+# functions for working with ini/cfg files
+#
+# This file is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+
+# global variables
+# -a -> indexed array
+# -A -> associative array
+declare -A cfg_repos
+declare -A cfg_file_map
+
+
+# functions
+function cfg_get_section {
+  if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then
+    echo ${BASH_REMATCH[2]}
+  else
+    echo ""
+  fi
+}
+
+function cfg_get_key_value {
+  if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then
+    echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}"
+  else
+    echo ""
+  fi
+}
+
+function cfg_load {
+  local cur_section=""
+  local cur_key=""
+  local cur_val=""
+  IFS=
+  while read -r line; do
+    new_section=$(cfg_get_section $line)
+    # got a new section
+    if [[ -n "$new_section" ]]; then
+      cur_section=$new_section
+    # not a section, try a key value
+    else
+      val=$(cfg_get_key_value $line)
+      # trim leading and trailing spaces as well
+      cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces)
+      cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces)
+      if [[ -n "$cur_key" ]]; then
+        # section + key is the associative in bash array, the field separator is space
+        cfg_repos[${cur_section} ${cur_key}]=$cur_val
+      fi
+    fi
+  done <$1
+}
+
+function cfg_print {
+  for index in "${!cfg_repos[@]}"
+  do
+    # split the associative key in to section and key
+    echo -n "section  : $(echo $index | cut -f1 -d ' ');"
+    echo -n "key  : $(echo $index | cut -f2 -d ' ');"
+    echo  "value: ${cfg_repos[$index]}"
+  done
+}
+
+function cfg_sections {
+  declare -A sections
+  for key in "${!cfg_repos[@]}"
+  do
+    # extract section from the associative key
+    section=$(echo $key | cut -f1 -d ' ')
+    sections[${section}]=1
+  done
+  for repo in "${!sections[@]}"
+  do
+      echo "${repo}"
+  done
+}
+
+function cfg_get_value {
+  section=$1
+  key=$2
+  echo "${cfg_repos[$section $key]}"
+}
+
+function cfg_trim_spaces {
+  # reads from argument $1 or stdin
+  if [[ $# -gt 0 ]]; then
+    sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< ${1}
+  else
+    sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' < /dev/stdin
+  fi
+}
+
+function cfg_trim_quotes {
+  # reads from argument $1 or stdin
+  if [[ $# -gt 0 ]]; then
+    sed -e 's/^"*//' -e 's/"*$//' <<< ${1}
+  else
+    sed -e 's/^"*//' -e 's/"*$//' < /dev/stdin
+  fi
+}
+
+function cfg_trim_curly_brackets {
+  # reads from argument $1 or stdin
+  if [[ $# -gt 0 ]]; then
+    sed -e 's/^{*//' -e 's/}*$//' <<< ${1}
+  else
+    sed -e 's/^{*//' -e 's/}*$//' < /dev/stdin
+  fi
+}
+
+function cfg_get_all_sections {
+  # first field in keys
+  # 1. get first field in all keys, 2. filter duplicates, 3. return them as string
+  declare -A all_sections
+  for key in "${!cfg_repos[@]}"
+  do
+    section=$(echo "$key" | cut -f1 -d' ')
+    all_sections[${section}]=1
+  done
+  sections=
+  for sec_key in "${!all_sections[@]}"
+  do
+    sections="${sections} ${sec_key}"
+  done
+  echo "${sections}" | cfg_trim_spaces
+}
+
+function cfg_init_file_map {
+  # strip '{' and '}' from config_map
+  # split config_map at ','
+  # for each item: split at ':' use first as key, second as value
+
+  # reset global variable
+  cfg_file_map=()
+
+  # expects a string containing the config_map from the cfg file
+  # trim leading and trailing curly brackets
+  cm_trimmed=$(cfg_trim_curly_brackets "$1")
+
+  # split into elements along ','
+  declare -a cm_mappings
+  IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}"
+
+  for index in "${!cm_mappings[@]}"
+  do
+    # split mapping into key and value
+    map_key=$(echo ${cm_mappings[index]} | cut -f1 -d':')
+    map_value=$(echo ${cm_mappings[index]} | cut -f2 -d':')
+    # trim spaces and double quotes at start and end
+    tr_key=$(cfg_trim_spaces "${map_key}" | cfg_trim_quotes)
+    tr_value=$(cfg_trim_spaces "${map_value}" | cfg_trim_quotes)
+    cfg_file_map[${tr_key}]=${tr_value}
+  done
+}
+
+function cfg_print_map {
+  for index in "${!cfg_file_map[@]}"
+  do
+    echo "${index} --> ${cfg_file_map[${index}]}"
+  done
+}
diff --git a/scripts/utils.sh b/scripts/utils.sh
new file mode 100644
index 00000000..baec35f0
--- /dev/null
+++ b/scripts/utils.sh
@@ -0,0 +1,117 @@
+function echo_green() {
+    echo -e "\e[32m$1\e[0m"
+}
+
+function echo_red() {
+    echo -e "\e[31m$1\e[0m"
+}
+
+function echo_yellow() {
+    echo -e "\e[33m$1\e[0m"
+}
+
+ANY_ERROR_EXITCODE=1
+function fatal_error() {
+    echo_red "ERROR: $1" >&2
+    if [[ $# -gt 1 ]]; then
+      exit $2
+    else
+      exit "${ANY_ERROR_EXITCODE}"
+    fi
+}
+
+function check_exit_code {
+    ec=$1
+    ok_msg=$2
+    fail_msg=$3
+
+    if [[ $ec -eq 0 ]]; then
+        echo_green "${ok_msg}"
+    else
+        fatal_error "${fail_msg}"
+    fi
+}
+
+function get_path_for_tool {
+    tool_name=$1
+    tool_envvar_name=$2
+
+    which_out=$(which ${tool_name} 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2
+        echo "${which_out}"
+        return 0
+    fi
+    if [[ -z "${tool_envvar_name}" ]]; then
+        msg="no env var holding the full path to tool '${tool_name}' provided"
+        echo "${msg}" >&2
+        return 1
+    else
+        tool_envvar_value=${!tool_envvar_name}
+        if [[ -x "${tool_envvar_value}" ]]; then
+            msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}"
+            echo "${msg}" >&2
+            echo "${tool_envvar_value}"
+            return 0
+        else
+            msg="ERROR: tool '${tool_name}' not in PATH\n"
+            msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH"
+            echo "${msg}" >&2
+            echo ""
+            return 2
+        fi
+    fi
+}
+
+function get_host_from_url {
+    url=$1
+    re="(http|https)://([^/:]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi
+}
+
+function get_port_from_url {
+    url=$1
+    re="(http|https)://[^:]+:([0-9]+)"
+    if [[ $url =~ $re ]]; then
+        echo ${BASH_REMATCH[2]}
+        return 0
+    else
+        echo ""
+        return 1
+    fi
+}
+
+function get_ipv4_address {
+    hname=$1
+    hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1)
+    # TODO try other methods if the one above does not work --> tool that verifies
+    #      what method can be used?
+    echo "${hipv4}"
+    return 0
+}
+
+# determine container runtime
+function get_container_runtime {
+    apptainer_out=$(which apptainer 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "${apptainer_out}"
+        return 0
+    fi
+    singularity_out=$(which singularity 2>&1)
+    exit_code=$?
+    if [[ ${exit_code} -eq 0 ]]; then
+        echo "${singularity_out}"
+        return 0
+    else
+        echo "false"
+        return 1
+    fi
+}

From d742027542ea7257afce0f42828e865daa0af87c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 11:54:41 +0200
Subject: [PATCH 02/25] updates to install_compatibility_layer.sh

- add arguments for resuming, retaining tmp and more verbose output
- extended and reformatted usage information
- source helper script scripts/utils.sh
- add support to resume a previous run given its temporary directory
- determine container runtime (singularity or apptainer)
- define env vars for both singularity and apptainer
- unset PKG_CONFIG_PATH before running ansible script
- only tar temporary directory if --retain-tmp option is given
- add message for how to resume a previous run
---
 install_compatibility_layer.sh | 101 +++++++++++++++++++++++++++------
 1 file changed, 85 insertions(+), 16 deletions(-)

diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh
index d01e2e18..7b28f1c2 100755
--- a/install_compatibility_layer.sh
+++ b/install_compatibility_layer.sh
@@ -7,22 +7,46 @@
 ARCH=
 CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian11
 REPOSITORY="pilot.eessi-hpc.org"
+RESUME=
+RETAIN_TMP=0
 STORAGE=
 VERSION=
+VERBOSE=
 
 display_help() {
   echo "usage: $0 [OPTIONS]"
-  echo " OPTIONS:"
-  echo "  -a | --arch ARCH       - architecture to build a compatibility layer for"
-  echo "                           [default/required: current host's architecture]"
-  echo "  -c | --container IMG   - image file or URL defining the container to use"
-  echo "                           [default: ${CONTAINER}"
-  echo "  -g | --storage DIR     - directory space on host machine (used for"
-  echo "                           temporary data) [default: 1. TMPDIR, 2. /tmp]"
-  echo "  -h | --help            - display this usage information"
-  echo "  -r | --repository REPO - CVMFS repository name [default: ${REPOSITORY}]"
-  echo "  -v | --version VERSION - override the EESSI stack version set in Ansible's"
-  echo "                           defaults/main.yml file [default: None]"
+  echo "OPTIONS:"
+  echo "    -a | --arch ARCHITECTURE"
+  echo "        architecture to build a compatibility layer for"
+  echo "        [default/required: current host's architecture]"
+  echo ""
+  echo "    -c | --container IMAGE"
+  echo "        image file or URL defining the container to use"
+  echo "        [default: ${CONTAINER}]"
+  echo ""
+  echo "    -g | --storage DIRECTORY"
+  echo "        directory space on host machine (used for"
+  echo "        temporary data) [default: 1. TMPDIR, 2. /tmp]"
+  echo ""
+  echo "    -k | --retain-tmp"
+  echo "        retain tmp storage (as tarball) for future"
+  echo "        inspection [default: not set]"
+  echo ""
+  echo "    -h | --help"
+  echo "        display this usage information"
+  echo ""
+  echo "    -r | --repository REPO"
+  echo "        CVMFS repository name [default: ${REPOSITORY}]"
+  echo ""
+  echo "    -t | --resume TMPDIR"
+  echo "        tmp directory to resume from [default: None]"
+  echo ""
+  echo "    -v | --version VERSION"
+  echo "        override the EESSI stack version set in Ansible's"
+  echo "        defaults/main.yml file [default: None]"
+  echo ""
+  echo "    --verbose"
+  echo "        increase verbosity of output [default: not set]"
   echo
 }
 
@@ -42,6 +66,10 @@ while [[ $# -gt 0 ]]; do
       STORAGE="$2"
       shift 2
       ;;
+    -k|--retain-tmp)
+      RETAIN_TMP=1
+      shift 1
+      ;;
     -h|--help)
       display_help
       exit 0
@@ -50,10 +78,18 @@ while [[ $# -gt 0 ]]; do
       REPOSITORY="$2"
       shift 2
       ;;
+    -t|--resume)
+      RESUME="$2"
+      shift 2
+      ;;
     -v|--version)
       VERSION="$2"
       shift 2
       ;;
+    --verbose)
+      VERBOSE="-vvv"
+      shift 1
+      ;;
     -*|--*)
       fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
       ;;
@@ -75,6 +111,9 @@ if [ ! -f "${SCRIPT_DIR}/ansible/playbooks/install.yml" ]; then
     exit 1
 fi
 
+# source utils.sh (for get_container_runtime and check_exit_code)
+source ${SCRIPT_DIR}/scripts/utils.sh
+
 # Check if the target architecture is set to the architecture of the current host,
 # as that's the only thing that's currently supported by this script
 HOST_ARCH=$(uname -m)
@@ -88,19 +127,38 @@ fi
 echo "A compatibility layer for architecture ${ARCH} will be built."
 
 # Make a temporary directory on the host for storing the installation and some temporary files
-TMPDIR=${STORAGE:-${TMPDIR:-/tmp}}
-mkdir -p ${TMPDIR}
-EESSI_TMPDIR=$(mktemp -d --tmpdir eessi.XXXXXXXXXX)
+if [[ ! -z ${RESUME} ]] && [[ -d ${RESUME} ]]; then
+    EESSI_TMPDIR=${RESUME}
+    echo "using previous temporary storage at ${RESUME} to resume work"
+else
+    TMPDIR=${STORAGE:-${TMPDIR:-/tmp}}
+    mkdir -p ${TMPDIR}
+    EESSI_TMPDIR=$(mktemp -d --tmpdir=${TMPDIR} eessi.XXXXXXXXXX)
+    echo "created new temporary storage at ${EESSI_TMPDIR}"
+fi
 echo "Using $EESSI_TMPDIR as temporary storage..."
 
 # Create temporary directories
 mkdir -p ${EESSI_TMPDIR}/cvmfs
 mkdir -p ${EESSI_TMPDIR}/home
 
+RUNTIME=$(get_container_runtime)
+exit_code=$?
+echo "RUNTIME='${RUNTIME}'"
+check_exit_code ${exit_code} "using runtime ${RUNTIME}" "oh no, neither apptainer nor singularity available"
+
 # Set up paths and mount points for Apptainer
-export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache
+if [[ -z ${APPTAINER_CACHEDIR} ]]; then
+  export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache
+fi
 export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer"
 export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
+# also define SINGULARITY_* env vars
+if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
+  export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache
+fi
+export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer"
+export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
 
 # Construct the Ansible playbook command
 ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=$(uname -m)"
@@ -110,11 +168,22 @@ fi
 if [[ ! -z ${REPOSITORY} ]]; then
     ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} -e cvmfs_repository=${REPOSITORY}"
 fi
+if [[ ! -z ${VERBOSE} ]]; then
+    ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} ${VERBOSE}"
+fi
 ANSIBLE_COMMAND="ansible-playbook ${ANSIBLE_OPTIONS} /compatibility-layer/ansible/playbooks/install.yml"
 # Finally, run Ansible inside the container to do the actual installation
 echo "Executing ${ANSIBLE_COMMAND} in ${CONTAINER}, this will take a while..."
-apptainer shell ${CONTAINER} <<EOF
+${RUNTIME} shell ${CONTAINER} <<EOF
 # The Gentoo Prefix bootstrap script will complain if $LD_LIBRARY_PATH is set
 unset LD_LIBRARY_PATH
+unset PKG_CONFIG_PATH
 ${ANSIBLE_COMMAND}
 EOF
+
+if [[ ${RETAIN_TMP} -eq 1 ]]; then
+  echo "Left container; tar'ing up ${EESSI_TMPDIR} for future inspection"
+  tar cvzf ${SCRIPT_DIR}/job_${SLURM_JOB_ID}_$(date +%s).tgz -C ${EESSI_TMPDIR} .
+fi
+
+echo "To resume work add '--resume ${EESSI_TMPDIR}'"

From 882e6b58ab4cb5fc9ea5989ae71b2aaf302eb56b Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 12:14:16 +0200
Subject: [PATCH 03/25] add default version + create own /tmp + fix id for tgz

---
 install_compatibility_layer.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh
index 7b28f1c2..76c56ff1 100755
--- a/install_compatibility_layer.sh
+++ b/install_compatibility_layer.sh
@@ -10,7 +10,7 @@ REPOSITORY="pilot.eessi-hpc.org"
 RESUME=
 RETAIN_TMP=0
 STORAGE=
-VERSION=
+VERSION=2023.04
 VERBOSE=
 
 display_help() {
@@ -141,6 +141,7 @@ echo "Using $EESSI_TMPDIR as temporary storage..."
 # Create temporary directories
 mkdir -p ${EESSI_TMPDIR}/cvmfs
 mkdir -p ${EESSI_TMPDIR}/home
+mkdir -p ${EESSI_TMPDIR}/tmp
 
 RUNTIME=$(get_container_runtime)
 exit_code=$?
@@ -158,6 +159,7 @@ if [[ -z ${SINGULARITY_CACHEDIR} ]]; then
   export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache
 fi
 export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer"
+export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/tmp:/tmp"
 export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
 
 # Construct the Ansible playbook command
@@ -183,7 +185,8 @@ EOF
 
 if [[ ${RETAIN_TMP} -eq 1 ]]; then
   echo "Left container; tar'ing up ${EESSI_TMPDIR} for future inspection"
-  tar cvzf ${SCRIPT_DIR}/job_${SLURM_JOB_ID}_$(date +%s).tgz -C ${EESSI_TMPDIR} .
+  ID=${SLURM_JOB_ID:-$$}
+  tar cvzf ${SCRIPT_DIR}/job_${ID}_$(date +%s).tgz -C ${EESSI_TMPDIR} .
 fi
 
 echo "To resume work add '--resume ${EESSI_TMPDIR}'"

From 0dfcf90186505ee1b99c0921b0a9a9e177808ee4 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 12:20:23 +0200
Subject: [PATCH 04/25] use version and repository from job.cfg or default +
 some cleanup

---
 bot/build.sh | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 6011cd87..fa09e21e 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -76,14 +76,14 @@ else
     echo "bot/build.sh: no modules to be loaded"
 fi
 
-#####################################################
-# cpu_target_arch=$(echo ${CPU_TARGET} | cut -d/ -f1)
 cpu_target_arch=$(cfg_get_value "architecture" "software_subdir" | cut -d/ -f1)
 host_arch=$(uname -m)
 eessi_arch=${cpu_target_arch:-${host_arch}}
 eessi_os=linux
-eessi_version=2023.04
-eessi_repo=pilot.eessi-hpc.org
+job_version=$(cfg_get_value "repository" "repo_version")
+eessi_version=${job_version:-2023.04}
+job_repo=$(cfg_get_value "repository" "repo_name")
+eessi_repo=${job_repo:-pilot.eessi-hpc.org}
 tar_topdir=/cvmfs/${eessi_repo}/versions
 
 if [ "${eessi_arch}" != "${host_arch}" ]; then
@@ -91,7 +91,6 @@ if [ "${eessi_arch}" != "${host_arch}" ]; then
   exit 1
 fi
 
-#./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -c ~/compat.sif
 ./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -g ${STORAGE}
 
 # create tarball -> should go into a separate script when this is supported by the bot

From 73af98898487d291981c0d0eb4aa89e2a2365725 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 12:46:37 +0200
Subject: [PATCH 05/25] bump version to 2023.04

---
 ansible/playbooks/roles/compatibility_layer/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
index cef05bac..e685a241 100644
--- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
+++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
@@ -1,6 +1,6 @@
 # Defaults file for the compatibility layer role.
 ---
-eessi_version: "2023.03"
+eessi_version: "2023.04"
 
 custom_overlays:
   - name: eessi

From cfc0604495e7bbbec7d14962fcfc4ff5d2a2bc05 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 12:48:55 +0200
Subject: [PATCH 06/25] replace community.general.portage with calling emerge
 in shell

---
 .../compatibility_layer/tasks/add_overlay.yml    | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
index 894d8392..8104391f 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
@@ -6,15 +6,15 @@
     creates: "{{ gentoo_prefix_path }}/usr/bin/equery"
 
 - name: Install eselect-repository
-  community.general.portage:
-    package: eselect-repository
-    state: present
+  ansible.builtin.command:
+    cmd: emerge app-eselect/eselect-repository
+    creates: "{{ gentoo_prefix_path }}/etc/eselect/repository.conf"
 
 # We need git in order to add Gentoo overlays hosted on git repositories.
 - name: Install git
-  community.general.portage:
-    package: dev-vcs/git
-    state: present
+  ansible.builtin.command:
+    cmd: emerge dev-vcs/git
+    creates: "{{ gentoo_prefix_path }}/usr/bin/git"
 
 - name: Check which repositories have been installed
   ansible.builtin.command: eselect repository list -i
@@ -39,8 +39,8 @@
       }}
 
 - name: Sync the repositories
-  community.general.portage:
-    sync: 'yes'
+  ansible.builtin.command:
+    cmd: emerge --sync
 
 - name: Find all files and directories in the etc/portage directory of the overlay
   ansible.builtin.find:

From 486257e729556aa739a873dbfcf18d0b63c41f46 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 12:51:18 +0200
Subject: [PATCH 07/25] add tasks to install and deselect build dependencies +
 use emerge to remove them

---
 .../roles/compatibility_layer/tasks/cleanup.yml   | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
index cc0a93bc..cee414b3 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
@@ -1,7 +1,16 @@
 # Clean up
 ---
+- name: Install redundant packages
+  ansible.builtin.command:
+    cmd: emerge --verbose "{{ item }}"
+  with_items: "{{ prefix_remove_packages }}"
+
+- name: Remove redundant packages from world set
+  ansible.builtin.command:
+    cmd: emerge --deselect --verbose "{{ item }}"
+  with_items: "{{ prefix_remove_packages }}"
+
 - name: Remove redundant packages
-  community.general.portage:
-    package: "{{ item }}"
-    state: absent
+  ansible.builtin.command:
+    cmd: emerge --unmerge --keep-going=y --verbose "{{ item }}"
   with_items: "{{ prefix_remove_packages }}"

From c4034a712fd3968f17f37379523374d21339fce9 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 12:53:22 +0200
Subject: [PATCH 08/25] use emerge to install package set (don't rebuild though
 and only install if list not empty)

---
 .../compatibility_layer/tasks/install_packages.yml    | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
index 4d0cb40e..b6eac7f1 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
@@ -1,15 +1,14 @@
 # Install a specified list of sets and packages.
 ---
 - name: Install package set {{ package_sets }}
-  community.general.portage:
-    package: "@{{ item }}"
-    state: present
+  ansible.builtin.command:
+    cmd: emerge --noreplace "@{{ item }}"
   with_items: "{{ package_sets }}"
   tags:
     - set
 
 - name: Install additional packages
-  community.general.portage:
-    package: "{{ item }}"
-    state: present
+  ansible.builtin.command:
+    cmd: emerge "{{ item }}"
   with_items: "{{ prefix_packages }}"
+  when: prefix_packages is iterable

From 6037645368954682b5edb0805621afe4bf5c3f98 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 12:57:05 +0200
Subject: [PATCH 09/25] provide missing profile for arm64 + make sure bash is
 used for bootstrapping tasks (because pipefail is needed)

---
 .../tasks/install_prefix.yml                  | 55 ++++++++++++++++++-
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml
index d9b4254d..c787c7e0 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml
@@ -135,6 +135,51 @@
   tags:
     - build_prefix
 
+- name: "Fix missing profile for arm64 - create directory"
+  ansible.builtin.file:
+    path: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64"
+    mode: '0755'
+    state: directory
+  when: eessi_host_arch == "aarch64"
+  tags:
+    - build_prefix
+
+- name: "Fix missing profile for arm64 - eapi"
+  ansible.builtin.copy:
+    content: "5"
+    dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64/eapi"
+    mode: 0644
+  when: eessi_host_arch == "aarch64"
+  tags:
+    - build_prefix
+
+- name: "Fix missing profile for arm64 - parent"
+  ansible.builtin.copy:
+    content: |
+      ../../../default/linux/arm64/17.0
+      ..
+    dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64/parent"
+    mode: 0644
+  when: eessi_host_arch == "aarch64"
+  tags:
+    - build_prefix
+
+- name: "Fix missing profile for arm64 - make.defaults"
+  ansible.builtin.copy:
+    content: |
+      # Copyright 1999-2018 Gentoo Foundation
+      # Distributed under the terms of the GNU General Public License v2
+
+      ACCEPT_KEYWORDS="arm64 ~arm64"
+
+      # Not sure if this is required as well, seems to be the default:
+      # CHOST=aarch64-unknown-linux-gnu
+    dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64/make.defaults"
+    mode: 0644
+  when: eessi_host_arch == "aarch64"
+  tags:
+    - build_prefix
+
 - name: "Mask packages for the bootstrap"
   ansible.builtin.copy:
     dest: "{{ gentoo_prefix_path }}/etc/portage/package.mask"
@@ -154,7 +199,10 @@
     - build_prefix
 
 - name: "Run Gentoo Prefix bootstrap stages 1-3 via {{ prefix_install }}"
-  ansible.builtin.shell: "set -o pipefail && {{ prefix_install }} | tee -a {{ prefix_build_log }} | grep -E '^(>>> Installing|\\* )'"
+  ansible.builtin.shell: set -o pipefail && ( {{ prefix_install }} | tee -a {{ prefix_build_log }} | grep -E '^(>>> Installing|\\* )' )
+  args:
+    executable: /bin/bash
+  become: false
   changed_when: true
   environment:
     STOP_BOOTSTRAP_AFTER: stage3
@@ -171,7 +219,10 @@
     - build_prefix
 
 - name: "Continue Gentoo Prefix bootstrap via {{ prefix_install }}"
-  ansible.builtin.shell: "set -o pipefail && {{ prefix_install }} | tee -a {{ prefix_build_log }} | grep -E '^(>>> Installing|\\* )'"
+  ansible.builtin.shell: set -o pipefail && ( {{ prefix_install }} | tee -a {{ prefix_build_log }} | grep -E '^(>>> Installing|\\* )' )
+  args:
+    executable: /bin/bash
+  become: false
   changed_when: true
   tags:
     - build_prefix

From 1db0017755302edb7ffba2576a17ac4523c10f33 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 13:02:15 +0200
Subject: [PATCH 10/25] replacing community.general.portage with emerge
 (noreplace false is default)

---
 .../compatibility_layer/tasks/set_glibc_trusted_dirs.yml    | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
index 80a5076f..fe414654 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
@@ -8,10 +8,8 @@
   when: eessi_host_os == "linux"
 
 - name: (Re)install glibc with the user-defined-trusted-dirs option
-  community.general.portage:
-    package: sys-libs/glibc
-    noreplace: false
-    oneshot: true
+  ansible.builtin.command:
+    cmd: emerge --oneshot sys-libs/glibc
   environment:
     EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
   when:

From f816a836b577a884e543083f3b19b9f495c3bbef Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 13:04:11 +0200
Subject: [PATCH 11/25] ensure ReFrame tests are working

- added some debugging
- remove any existing virtualenv
- make sure ReFrame is installed
- left some more example debugging (could be cleaned up)
---
 .../roles/compatibility_layer/tasks/test.yml  | 66 +++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
index 631ad3e5..c7bdb792 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -10,11 +10,77 @@
   tags:
     - test
 
+- name: Check for python3
+  ansible.builtin.command:
+    cmd: "which python3"
+  register: which_python3
+
+- debug: msg="{{ which_python3.stdout }}"
+
+- name: "Check if {{ reframe_venv_dir }} exists"
+  ansible.builtin.command:
+    cmd: "ls -l {{ reframe_venv_dir }}"
+  register: ls_reframe_venv_dir
+  ignore_errors: true
+  changed_when: false
+  tags:
+    - test
+
+- debug: msg="{{ ls_reframe_venv_dir }}"
+
+#- name: "Show contents of {{ reframe_venv_dir }}/bin"
+#  ansible.builtin.command:
+#    cmd: "ls -l {{ reframe_venv_dir }}/bin"
+#  register: ls_reframe_venv_dir_bin
+#  ignore_errors: true
+#  changed_when: false
+#  tags:
+#    - test
+#
+#- debug: msg="{{ ls_reframe_venv_dir_bin }}"
+
+- name: Remove venv
+  ansible.builtin.shell: |
+    rm -rf "{{ reframe_venv_dir }}"
+  when: reframe_exists.rc != 0
+  register: remove_venv
+  ignore_errors: true
+  changed_when: false
+  tags:
+    - test
+
+- debug: msg="{{ remove_venv.stdout }}"
+
+#- name: "Show contents of {{ reframe_venv_dir }}"
+#  ansible.builtin.command:
+#    cmd: "ls -l {{ reframe_venv_dir }}"
+#  register: ls_reframe_venv_dir
+#  ignore_errors: true
+#  changed_when: false
+#  tags:
+#    - test
+#
+#- debug: msg="{{ ls_reframe_venv_dir }}"
+
+#- name: Install ReFrame with venv pip
+#  ansible.builtin.shell: |
+#    which python3
+#    #python3 -m venv "{{ reframe_venv_dir }}"
+#    #. "{{ reframe_venv_dir }}/bin/activate"
+#    #which python3
+#    #    python3 -m pip install ReFrame-HPC
+#  when: reframe_exists.rc != 0
+#  register: pip_reframe
+#
+#- debug: msg="{{ pip_reframe.stdout }}"
+
 - name: Install Reframe using pip if it's not installed yet
   ansible.builtin.pip:
     name: ReFrame-HPC
     virtualenv: "{{ reframe_venv_dir }}"
+    #    virtualenv_command: "{{ gentoo_prefix_path }}/usr/bin/python3 -m venv"
     virtualenv_command: python3 -m venv
+    state: forcereinstall
   when: reframe_exists.rc != 0
 
 - name: Copy ReFrame test file

From 0a5f11c42f2c7994f922beeac85d3e8d1b70cd9d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 13:54:40 +0200
Subject: [PATCH 12/25] attempt to fix Ansible lint issues

---
 .../compatibility_layer/tasks/add_overlay.yml |  2 ++
 .../compatibility_layer/tasks/cleanup.yml     |  6 +++++
 .../tasks/install_packages.yml                |  4 ++++
 .../roles/compatibility_layer/tasks/test.yml  | 23 +++++++++++--------
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
index 8104391f..8e67071f 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
@@ -41,6 +41,8 @@
 - name: Sync the repositories
   ansible.builtin.command:
     cmd: emerge --sync
+  register: sync_repos
+  changed_when: sync_repo.rc != 0
 
 - name: Find all files and directories in the etc/portage directory of the overlay
   ansible.builtin.find:
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
index cee414b3..778b3513 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
@@ -3,14 +3,20 @@
 - name: Install redundant packages
   ansible.builtin.command:
     cmd: emerge --verbose "{{ item }}"
+  register: emerge_redundant
   with_items: "{{ prefix_remove_packages }}"
+  changed_when: emerge_redundant.rc != 0
 
 - name: Remove redundant packages from world set
   ansible.builtin.command:
     cmd: emerge --deselect --verbose "{{ item }}"
+  register: deselect_redundant
   with_items: "{{ prefix_remove_packages }}"
+  changed_when: deselect_redundant.rc != 0
 
 - name: Remove redundant packages
   ansible.builtin.command:
     cmd: emerge --unmerge --keep-going=y --verbose "{{ item }}"
+  register: unmerge_redundant
   with_items: "{{ prefix_remove_packages }}"
+  changed_when: unmerge_redundant.rc != 0
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
index b6eac7f1..a5d700a5 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
@@ -3,12 +3,16 @@
 - name: Install package set {{ package_sets }}
   ansible.builtin.command:
     cmd: emerge --noreplace "@{{ item }}"
+  register: emerge_item_from_set
   with_items: "{{ package_sets }}"
   tags:
     - set
+  changed_when: emerge_item_from_set.rc != 0
 
 - name: Install additional packages
   ansible.builtin.command:
     cmd: emerge "{{ item }}"
+  register: emerge_additional_item
   with_items: "{{ prefix_packages }}"
   when: prefix_packages is iterable
+  changed_when: emerge_additional_item.rc != 0
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
index c7bdb792..09066941 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -15,9 +15,10 @@
     cmd: "which python3"
   register: which_python3
 
-- debug: msg="{{ which_python3.stdout }}"
+- name: Result for python3 check
+  ansible.builtin.debug: msg="{{ which_python3.stdout }}"
 
-- name: "Check if {{ reframe_venv_dir }} exists"
+- name: "Check if venv exists {{ reframe_venv_dir }}"
   ansible.builtin.command:
     cmd: "ls -l {{ reframe_venv_dir }}"
   register: ls_reframe_venv_dir
@@ -26,9 +27,10 @@
   tags:
     - test
 
-- debug: msg="{{ ls_reframe_venv_dir }}"
+- name: "Result for checking {{ reframe_venv_dir }}"
+  ansible.builtin.debug: msg="{{ ls_reframe_venv_dir }}"
 
-#- name: "Show contents of {{ reframe_venv_dir }}/bin"
+# - name: "Show contents of {{ reframe_venv_dir }}/bin"
 #  ansible.builtin.command:
 #    cmd: "ls -l {{ reframe_venv_dir }}/bin"
 #  register: ls_reframe_venv_dir_bin
@@ -37,7 +39,7 @@
 #  tags:
 #    - test
 #
-#- debug: msg="{{ ls_reframe_venv_dir_bin }}"
+# - debug: msg="{{ ls_reframe_venv_dir_bin }}"
 
 - name: Remove venv
   ansible.builtin.shell: |
@@ -49,9 +51,10 @@
   tags:
     - test
 
-- debug: msg="{{ remove_venv.stdout }}"
+- name: Result for removing venv
+  ansible.builtin.debug: msg="{{ remove_venv.stdout }}"
 
-#- name: "Show contents of {{ reframe_venv_dir }}"
+# - name: "Show contents of {{ reframe_venv_dir }}"
 #  ansible.builtin.command:
 #    cmd: "ls -l {{ reframe_venv_dir }}"
 #  register: ls_reframe_venv_dir
@@ -60,9 +63,9 @@
 #  tags:
 #    - test
 #
-#- debug: msg="{{ ls_reframe_venv_dir }}"
+# - debug: msg="{{ ls_reframe_venv_dir }}"
 
-#- name: Install ReFrame with venv pip
+# - name: Install ReFrame with venv pip
 #  ansible.builtin.shell: |
 #    which python3
 #    #python3 -m venv "{{ reframe_venv_dir }}"
@@ -72,7 +75,7 @@
 #  when: reframe_exists.rc != 0
 #  register: pip_reframe
 #
-#- debug: msg="{{ pip_reframe.stdout }}"
+# - debug: msg="{{ pip_reframe.stdout }}"
 
 - name: Install Reframe using pip if it's not installed yet
   ansible.builtin.pip:

From a7561413cb248d6cb2d853f3059d4e75620639a8 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 13:59:38 +0200
Subject: [PATCH 13/25] attempt to fix Ansible lint issues, part 2

---
 .../playbooks/roles/compatibility_layer/tasks/test.yml    | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
index 09066941..19156e88 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -14,9 +14,11 @@
   ansible.builtin.command:
     cmd: "which python3"
   register: which_python3
+  changed_when: which_python3.rc != 0
 
 - name: Result for python3 check
   ansible.builtin.debug: msg="{{ which_python3.stdout }}"
+  changed_when: false
 
 - name: "Check if venv exists {{ reframe_venv_dir }}"
   ansible.builtin.command:
@@ -29,6 +31,7 @@
 
 - name: "Result for checking {{ reframe_venv_dir }}"
   ansible.builtin.debug: msg="{{ ls_reframe_venv_dir }}"
+  changed_when: false
 
 # - name: "Show contents of {{ reframe_venv_dir }}/bin"
 #  ansible.builtin.command:
@@ -42,8 +45,8 @@
 # - debug: msg="{{ ls_reframe_venv_dir_bin }}"
 
 - name: Remove venv
-  ansible.builtin.shell: |
-    rm -rf "{{ reframe_venv_dir }}"
+  ansible.builtin.command:
+    cmd: "rm -rf {{ reframe_venv_dir }}"
   when: reframe_exists.rc != 0
   register: remove_venv
   ignore_errors: true
@@ -53,6 +56,7 @@
 
 - name: Result for removing venv
   ansible.builtin.debug: msg="{{ remove_venv.stdout }}"
+  changed_when: false
 
 # - name: "Show contents of {{ reframe_venv_dir }}"
 #  ansible.builtin.command:

From 3173d4cf5163e142af720920410c8620f5bc0a9e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 14:04:41 +0200
Subject: [PATCH 14/25] attempt to fix Ansible lint issues, part 3

---
 ansible/playbooks/roles/compatibility_layer/tasks/test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
index 19156e88..a5c939de 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -17,7 +17,8 @@
   changed_when: which_python3.rc != 0
 
 - name: Result for python3 check
-  ansible.builtin.debug: msg="{{ which_python3.stdout }}"
+  ansible.builtin.debug:
+    msg: "{{ which_python3.stdout }}"
   changed_when: false
 
 - name: "Check if venv exists {{ reframe_venv_dir }}"

From ae8ea3d30eeeeafa0e4291683fd9fe7bc51ae0de Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 14:47:46 +0200
Subject: [PATCH 15/25] attempt to fix Ansible lint issues, part 4

---
 .../roles/compatibility_layer/tasks/test.yml     | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
index a5c939de..2a1fbd4c 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -31,7 +31,8 @@
     - test
 
 - name: "Result for checking {{ reframe_venv_dir }}"
-  ansible.builtin.debug: msg="{{ ls_reframe_venv_dir }}"
+  ansible.builtin.debug:
+    msg: "{{ ls_reframe_venv_dir }}"
   changed_when: false
 
 # - name: "Show contents of {{ reframe_venv_dir }}/bin"
@@ -46,17 +47,16 @@
 # - debug: msg="{{ ls_reframe_venv_dir_bin }}"
 
 - name: Remove venv
-  ansible.builtin.command:
-    cmd: "rm -rf {{ reframe_venv_dir }}"
-  when: reframe_exists.rc != 0
+  ansible.builtin.file:
+    path: "{{ reframe_venv_dir }}"
+    state: absent
+    recursive: yes
   register: remove_venv
   ignore_errors: true
-  changed_when: false
-  tags:
-    - test
 
 - name: Result for removing venv
-  ansible.builtin.debug: msg="{{ remove_venv.stdout }}"
+  ansible.builtin.debug:
+    msg: "{{ remove_venv.stdout }}"
   changed_when: false
 
 # - name: "Show contents of {{ reframe_venv_dir }}"

From 2648ddfb63f2f02c84ea9c97c4af2e6b3091d0eb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 14:53:48 +0200
Subject: [PATCH 16/25] attempt to fix Ansible lint issues, part 5

---
 ansible/playbooks/roles/compatibility_layer/tasks/test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
index 2a1fbd4c..351e5b67 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -50,7 +50,7 @@
   ansible.builtin.file:
     path: "{{ reframe_venv_dir }}"
     state: absent
-    recursive: yes
+    recursive: true
   register: remove_venv
   ignore_errors: true
 

From f92e9561f199bf33f6bc7209cb32ca0b8926525d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 19:13:09 +0200
Subject: [PATCH 17/25] fix typo

---
 .../playbooks/roles/compatibility_layer/tasks/add_overlay.yml   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
index 8e67071f..1843e5f5 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
@@ -42,7 +42,7 @@
   ansible.builtin.command:
     cmd: emerge --sync
   register: sync_repos
-  changed_when: sync_repo.rc != 0
+  changed_when: sync_repos.rc != 0
 
 - name: Find all files and directories in the etc/portage directory of the overlay
   ansible.builtin.find:

From 6b2cf67937d9f76e20722a2fb6b9afea9675d106 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 19:19:35 +0200
Subject: [PATCH 18/25] add arg to retain eessi_tmpdir + improved messages

---
 bot/build.sh                   | 5 +++--
 install_compatibility_layer.sh | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index fa09e21e..9995db52 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -91,7 +91,8 @@ if [ "${eessi_arch}" != "${host_arch}" ]; then
   exit 1
 fi
 
-./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -g ${STORAGE}
+# option -k is used for retaining ${eessi_tmp}
+./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -g ${STORAGE} -k
 
 # create tarball -> should go into a separate script when this is supported by the bot
 target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz
@@ -100,6 +101,6 @@ if [ -d ${eessi_tmp}/${tar_topdir}/${eessi_version} ]; then
   tar cfvz ${target_tgz} -C ${eessi_tmp}/${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch}
   echo ${target_tgz} created!
 else
-  echo "Directory ${tar_topdir}/${eessi_version} was not created, not creating tarball."
+  echo "Directory ${eessi_tmp}/${tar_topdir}/${eessi_version} was not created, not creating tarball."
   exit 1
 fi
diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh
index 76c56ff1..0be27162 100755
--- a/install_compatibility_layer.sh
+++ b/install_compatibility_layer.sh
@@ -186,7 +186,10 @@ EOF
 if [[ ${RETAIN_TMP} -eq 1 ]]; then
   echo "Left container; tar'ing up ${EESSI_TMPDIR} for future inspection"
   ID=${SLURM_JOB_ID:-$$}
-  tar cvzf ${SCRIPT_DIR}/job_${ID}_$(date +%s).tgz -C ${EESSI_TMPDIR} .
+  TIMESTAMP=$(date +%s)
+  TGZ=${SCRIPT_DIR}/job_${ID}_${TIMESTAMP}.tgz
+  tar cvzf ${TGZ} -C ${EESSI_TMPDIR} .
+  echo "created tarball '${TGZ}'"
 fi
 
 echo "To resume work add '--resume ${EESSI_TMPDIR}'"

From 61997e15f2c57ad115c9b9633c6b9fe421371fc6 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 3 Apr 2023 21:24:41 +0200
Subject: [PATCH 19/25] add missing APPTAINER_BIND

---
 install_compatibility_layer.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh
index 0be27162..84e8edee 100755
--- a/install_compatibility_layer.sh
+++ b/install_compatibility_layer.sh
@@ -153,6 +153,7 @@ if [[ -z ${APPTAINER_CACHEDIR} ]]; then
   export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache
 fi
 export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer"
+export APPTAINER_BIND="${APPTAINER_BIND},${EESSI_TMPDIR}/tmp:/tmp"
 export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}"
 # also define SINGULARITY_* env vars
 if [[ -z ${SINGULARITY_CACHEDIR} ]]; then

From 016509ae7f1b2fb1bc7794a5b13ccb116ab3d2d8 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 4 Apr 2023 07:29:33 +0200
Subject: [PATCH 20/25] a few fixes + removing commented out code

---
 .../roles/compatibility_layer/tasks/test.yml  | 40 ++-----------------
 1 file changed, 3 insertions(+), 37 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
index 351e5b67..ad03b635 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml
@@ -35,58 +35,24 @@
     msg: "{{ ls_reframe_venv_dir }}"
   changed_when: false
 
-# - name: "Show contents of {{ reframe_venv_dir }}/bin"
-#  ansible.builtin.command:
-#    cmd: "ls -l {{ reframe_venv_dir }}/bin"
-#  register: ls_reframe_venv_dir_bin
-#  ignore_errors: true
-#  changed_when: false
-#  tags:
-#    - test
-#
-# - debug: msg="{{ ls_reframe_venv_dir_bin }}"
-
 - name: Remove venv
   ansible.builtin.file:
     path: "{{ reframe_venv_dir }}"
     state: absent
-    recursive: true
   register: remove_venv
+  when: ls_reframe_venv_dir.rc == 0
   ignore_errors: true
 
 - name: Result for removing venv
   ansible.builtin.debug:
-    msg: "{{ remove_venv.stdout }}"
+    msg: "{{ remove_venv }}"
+  when: ls_reframe_venv_dir.rc == 0
   changed_when: false
 
-# - name: "Show contents of {{ reframe_venv_dir }}"
-#  ansible.builtin.command:
-#    cmd: "ls -l {{ reframe_venv_dir }}"
-#  register: ls_reframe_venv_dir
-#  ignore_errors: true
-#  changed_when: false
-#  tags:
-#    - test
-#
-# - debug: msg="{{ ls_reframe_venv_dir }}"
-
-# - name: Install ReFrame with venv pip
-#  ansible.builtin.shell: |
-#    which python3
-#    #python3 -m venv "{{ reframe_venv_dir }}"
-#    #. "{{ reframe_venv_dir }}/bin/activate"
-#    #which python3
-#    #    python3 -m pip install ReFrame-HPC
-#  when: reframe_exists.rc != 0
-#  register: pip_reframe
-#
-# - debug: msg="{{ pip_reframe.stdout }}"
-
 - name: Install Reframe using pip if it's not installed yet
   ansible.builtin.pip:
     name: ReFrame-HPC
     virtualenv: "{{ reframe_venv_dir }}"
-    #    virtualenv_command: "{{ gentoo_prefix_path }}/usr/bin/python3 -m venv"
     virtualenv_command: python3 -m venv
     state: forcereinstall
   when: reframe_exists.rc != 0

From 7bfa90de4f9f2014c57eb78f9ea6a640c12f53df Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Tue, 4 Apr 2023 16:52:57 +0000
Subject: [PATCH 21/25] reverted removal of community.general.portage

added

```ansible
  vars:
   ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
```

to all tasks using communit.general.portage
---
 .../compatibility_layer/tasks/add_overlay.yml | 35 +++++++++++-----
 .../compatibility_layer/tasks/cleanup.yml     | 40 +++++++++++--------
 .../tasks/install_packages.yml                | 32 ++++++++++-----
 .../tasks/set_glibc_trusted_dirs.yml          | 15 ++++++-
 4 files changed, 84 insertions(+), 38 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
index 1843e5f5..259785ed 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
@@ -6,15 +6,25 @@
     creates: "{{ gentoo_prefix_path }}/usr/bin/equery"
 
 - name: Install eselect-repository
-  ansible.builtin.command:
-    cmd: emerge app-eselect/eselect-repository
-    creates: "{{ gentoo_prefix_path }}/etc/eselect/repository.conf"
+  community.general.portage:
+    package: eselect-repository
+    state: present
+  vars:
+    ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
+#  ansible.builtin.command:
+#    cmd: emerge app-eselect/eselect-repository
+#    creates: "{{ gentoo_prefix_path }}/etc/eselect/repository.conf"
 
 # We need git in order to add Gentoo overlays hosted on git repositories.
 - name: Install git
-  ansible.builtin.command:
-    cmd: emerge dev-vcs/git
-    creates: "{{ gentoo_prefix_path }}/usr/bin/git"
+  community.general.portage:
+    package: dev-vcs/git
+    state: present
+  vars:
+    ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
+#  ansible.builtin.command:
+#    cmd: emerge dev-vcs/git
+#    creates: "{{ gentoo_prefix_path }}/usr/bin/git"
 
 - name: Check which repositories have been installed
   ansible.builtin.command: eselect repository list -i
@@ -39,10 +49,15 @@
       }}
 
 - name: Sync the repositories
-  ansible.builtin.command:
-    cmd: emerge --sync
-  register: sync_repos
-  changed_when: sync_repos.rc != 0
+  community.general.portage:
+    sync: 'yes'
+    verbose: true
+  vars:
+    ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
+#  ansible.builtin.command:
+#    cmd: emerge --sync
+#  register: sync_repos
+#  changed_when: sync_repos.rc != 0
 
 - name: Find all files and directories in the etc/portage directory of the overlay
   ansible.builtin.find:
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
index 778b3513..73cceb91 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
@@ -1,22 +1,28 @@
 # Clean up
 ---
-- name: Install redundant packages
-  ansible.builtin.command:
-    cmd: emerge --verbose "{{ item }}"
-  register: emerge_redundant
-  with_items: "{{ prefix_remove_packages }}"
-  changed_when: emerge_redundant.rc != 0
-
-- name: Remove redundant packages from world set
-  ansible.builtin.command:
-    cmd: emerge --deselect --verbose "{{ item }}"
-  register: deselect_redundant
-  with_items: "{{ prefix_remove_packages }}"
-  changed_when: deselect_redundant.rc != 0
+#- name: Install redundant packages
+#  ansible.builtin.command:
+#    cmd: emerge --verbose "{{ item }}"
+#  register: emerge_redundant
+#  with_items: "{{ prefix_remove_packages }}"
+#  changed_when: emerge_redundant.rc != 0
+#
+#- name: Remove redundant packages from world set
+#  ansible.builtin.command:
+#    cmd: emerge --deselect --verbose "{{ item }}"
+#  register: deselect_redundant
+#  with_items: "{{ prefix_remove_packages }}"
+#  changed_when: deselect_redundant.rc != 0
 
 - name: Remove redundant packages
-  ansible.builtin.command:
-    cmd: emerge --unmerge --keep-going=y --verbose "{{ item }}"
-  register: unmerge_redundant
+  community.general.portage:
+    package: "{{ item }}"
+    state: absent
   with_items: "{{ prefix_remove_packages }}"
-  changed_when: unmerge_redundant.rc != 0
+  vars:
+    ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
+#  ansible.builtin.command:
+#    cmd: emerge --unmerge --keep-going=y --verbose "{{ item }}"
+#  register: unmerge_redundant
+#  with_items: "{{ prefix_remove_packages }}"
+#  changed_when: unmerge_redundant.rc != 0
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
index a5d700a5..df5f05b1 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
@@ -1,18 +1,32 @@
 # Install a specified list of sets and packages.
 ---
 - name: Install package set {{ package_sets }}
-  ansible.builtin.command:
-    cmd: emerge --noreplace "@{{ item }}"
-  register: emerge_item_from_set
+  community.general.portage:
+    package: "@{{ item }}"
+    state: present
   with_items: "{{ package_sets }}"
+  vars:
+    ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
   tags:
     - set
-  changed_when: emerge_item_from_set.rc != 0
+#  ansible.builtin.command:
+#    cmd: emerge --noreplace "@{{ item }}"
+#  register: emerge_item_from_set
+#  with_items: "{{ package_sets }}"
+#  tags:
+#    - set
+#  changed_when: emerge_item_from_set.rc != 0
 
 - name: Install additional packages
-  ansible.builtin.command:
-    cmd: emerge "{{ item }}"
-  register: emerge_additional_item
+  community.general.portage:
+    package: "{{ item }}"
+    state: present
   with_items: "{{ prefix_packages }}"
-  when: prefix_packages is iterable
-  changed_when: emerge_additional_item.rc != 0
+  vars:
+    ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
+#  ansible.builtin.command:
+#    cmd: emerge "{{ item }}"
+#  register: emerge_additional_item
+#  with_items: "{{ prefix_packages }}"
+#  when: prefix_packages is iterable
+#  changed_when: emerge_additional_item.rc != 0
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
index fe414654..8710960b 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
@@ -8,13 +8,24 @@
   when: eessi_host_os == "linux"
 
 - name: (Re)install glibc with the user-defined-trusted-dirs option
-  ansible.builtin.command:
-    cmd: emerge --oneshot sys-libs/glibc
+  community.general.portage:
+    package: sys-libs/glibc
+    noreplace: false
+    oneshot: true
   environment:
     EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
+  vars:
+    ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
   when:
     - eessi_host_os == "linux"
     - glibc_extra_emake.stdout != "user-defined-trusted-dirs=" + ":".join(prefix_user_defined_trusted_dirs)
+#  ansible.builtin.command:
+#    cmd: emerge --oneshot sys-libs/glibc
+#  environment:
+#    EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
+#  when:
+#    - eessi_host_os == "linux"
+#    - glibc_extra_emake.stdout != "user-defined-trusted-dirs=" + ":".join(prefix_user_defined_trusted_dirs)
 
 - name: Create portage env directory
   ansible.builtin.file:

From 8f5f5fab324dfe8c9e20f1ad0d1275d51ddf01d5 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 9 Apr 2023 20:34:58 +0000
Subject: [PATCH 22/25] test commit from March 13

---
 .../compatibility_layer/defaults/main.yml     | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
index e685a241..c87e83c7 100644
--- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
+++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
@@ -5,7 +5,7 @@ eessi_version: "2023.04"
 custom_overlays:
   - name: eessi
     source: git
-    url: https://github.com/EESSI/gentoo-overlay.git
+    url: https://github.com/trz42/gentoo-overlay.git
     eclass-overrides: true
 
 cvmfs_repository: pilot.eessi-hpc.org
@@ -16,7 +16,16 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/c
 gentoo_git_repo: https://github.com/gentoo/gentoo.git
 # Select a specific commit in the gentoo_git_repo that should be used for the bootstrap,
 # e.g. by checking: https://github.com/gentoo/gentoo/commits/master
-gentoo_git_commit: 4ca74e7abe4f2b14e686267b517c59d43bb580b4
+# March 7 (4ca74e7abe4f2b14e686267b517c59d43bb580b4)
+# gentoo_git_commit: 4ca74e7abe4f2b14e686267b517c59d43bb580b4
+# April 4 (b9ebfbaef10f4740671c8bd90c6d2c929f8379bc) --> failed with circular dependencies (cmake, curl, nghttp2), cmake was update early on April 5
+# gentoo_git_commit: b9ebfbaef10f4740671c8bd90c6d2c929f8379bc
+# April 5 (8c0ce1bcd089d5badcb39afce98111bf4cfbc8f0)
+# gentoo_git_commit: 8c0ce1bcd089d5badcb39afce98111bf4cfbc8f0
+# March 13 (7db97f5a3fc7ca8eced2a94e5f9481f7556cf908) one commit before 3a38f9b245e674722d5d58dfbb6baafca2b26975 (which introduced nghttp2 1.52.0 which switched to using cmake)
+# gentoo_git_commit: 7db97f5a3fc7ca8eced2a94e5f9481f7556cf908
+# March 13 (3a38f9b245e674722d5d58dfbb6baafca2b26975) commit which introduced nghttp2 1.52.0 which switched to using cmake
+gentoo_git_commit: 3a38f9b245e674722d5d58dfbb6baafca2b26975
 prefix_required_space: 15 GB
 prefix_default_gcc: 9.5.0
 prefix_user_defined_trusted_dirs:
@@ -25,6 +34,12 @@ prefix_mask_packages: |
   # stick to GCC 9.x; using a too recent compiler in the compat layer complicates stuff in the software layer,
   # see for example https://github.com/EESSI/software-layer/issues/151
   >=sys-devel/gcc-10
+  # it looks like sys-apps/portaga-3.0.45.3-r2 creates a circular dependency that wasn't there before (version 3.0.45.2 with commit from March 7)
+  # didn't solve the problem with circular dependencies
+  # >=sys-apps/portage-3.0.45.3  # it looks like sys-apps/portaga-3.0.45.3-r2 creates a circular dependency that wasn't there before (version 3.0.45.2 with commit from March 7)
+  # net-libs/nghttp2-1.52.0 was introduced on March 13 (3a38f9b245e674722d5d58dfbb6baafca2b26975) switching to cmake ...
+  # .. instead of using a commit before (7db97f5a3fc7ca8eced2a94e5f9481f7556cf908) here we try to mask any version equal or younger
+  # >= net-libs/nghttp2-1.52.0
 prefix_unmask_packages: |
   # unmask older GCC to make it installable
   =sys-devel/gcc-9*

From 1cc55a096afcc034b680941589714b4f364737b8 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 9 Apr 2023 21:59:19 +0000
Subject: [PATCH 23/25] initial version of script to check result of building
 compat layer

---
 bot/check-result.sh | 218 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 218 insertions(+)
 create mode 100755 bot/check-result.sh

diff --git a/bot/check-result.sh b/bot/check-result.sh
new file mode 100755
index 00000000..28639174
--- /dev/null
+++ b/bot/check-result.sh
@@ -0,0 +1,218 @@
+#!/bin/bash
+#
+# Script to check the result of building the EESSI compatibility layer.
+# Intended use is that it is called by a (batch) job running on a compute
+# node.
+#
+# This script is part of the EESSI compatibility layer, see
+# https://github.com/EESSI/compatibility-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files referenced in job.cfg
+
+# Example output
+# beginning of job output
+#### A compatibility layer for architecture x86_64 will be built.
+#### created new temporary storage at /srv/eessi-2023.04/TS/eessi.fjgC41DsgS
+#### Using /srv/eessi-2023.04/TS/eessi.fjgC41DsgS as temporary storage...
+#### RUNTIME='/usr/bin/apptainer'
+#### ESC[32musing runtime /usr/bin/apptainerESC[0m
+#### Executing ansible-playbook -e eessi_host_os=linux -e eessi_host_arch=x86_64 -e eessi_version=2023.04 -e cvmfs_repository=pilot.eessi-hpc.org /compatibility-layer/ansible/playbooks/install.yml in docker://ghcr.io/eessi/bootstrap-prefix:debian11, this will take a while.
+
+# good TASKs to check for
+#### TASK [compatibility_layer : Create Gentoo prefix path and log directory] *******
+#### changed: [localhost] => (item=/cvmfs/pilot.eessi-hpc.org/versions/2023.04/compat/linux/x86_64)
+#### changed: [localhost] => (item=/tmp/eessi-logs)
+#### 
+#### TASK [compatibility_layer : Add custom overlay configuration] ******************
+#### skipping: [localhost] => (item={'name': 'eessi', 'source': 'git', 'url': 'https://github.com/trz42/gentoo-overlay.git', 'eclass-overrides': True}) 
+#### skipping: [localhost]
+#### 
+#### TASK [compatibility_layer : Make configuration file with overlays that can override eclasses] ***
+#### ok: [localhost]
+#### 
+#### TASK [compatibility_layer : Sync the repositories] *****************************
+#### ok: [localhost]
+#### 
+#### TASK [compatibility_layer : Run Gentoo Prefix bootstrap stages 1-3 via /tmp/bootstrap-prefix.sh /cvmfs/pilot.eessi-hpc.org/versions/2023.04/compat/linux/x86_64 noninteractive] ***
+#### changed: [localhost]
+#### 
+#### TASK [compatibility_layer : Specify use flags before completing bootstrap] *****
+#### changed: [localhost]
+#### 
+#### TASK [compatibility_layer : Continue Gentoo Prefix bootstrap via /tmp/bootstrap-prefix.sh /cvmfs/pilot.eessi-hpc.org/versions/2023.04/compat/linux/x86_64 noninteractive] ***
+#### changed: [localhost]
+#### 
+#### TASK [compatibility_layer : (Re)install glibc with the user-defined-trusted-dirs option] ***
+#### skipping: [localhost]
+#### 
+#### TASK [compatibility_layer : Create portage env directory] **********************
+#### ok: [localhost]
+#### 
+#### TASK [compatibility_layer : Add env file for glibc to make sure the user-defined-trusted-dirs is always used] ***
+#### ok: [localhost]
+#### 
+#### TASK [compatibility_layer : Install package set ['eessi-2023.04-linux-x86_64']] ***
+#### ok: [localhost] => (item=eessi-2023.04-linux-x86_64)
+#### 
+#### TASK [compatibility_layer : Remove redundant packages] *************************
+#### ok: [localhost] => (item=dev-lang/go)
+#### ok: [localhost] => (item=dev-lang/go-bootstrap)
+#### 
+#### TASK [compatibility_layer : Run ReFrame tests] *********************************
+#### ok: [localhost]
+
+# end of job output
+#### PLAY RECAP *********************************************************************
+#### localhost                  : ok=17   changed=13   unreachable=0    failed=1    skipped=6    rescued=0    ignored=0   
+#### localhost                  : ok=38   changed=3    unreachable=0    failed=0    skipped=5    rescued=0    ignored=1   
+
+
+# stop as soon as something fails
+# set -e
+
+# TODO decide later what we actually need (scripts and cfg values)
+# source utils.sh and cfg_files.sh
+##source scripts/utils.sh
+##source scripts/cfg_files.sh
+##
+### defaults
+##export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+##
+### check if ${JOB_CFG_FILE} exists
+##if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+##    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+##fi
+##echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side"
+##cat ${JOB_CFG_FILE}
+##
+##echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+##cfg_load ${JOB_CFG_FILE}
+##
+##LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
+##echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
+##
+##echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+### replace any env variable in ${LOCAL_TMP} with its
+###   current value (e.g., a value that is local to the job)
+##STORAGE=$(envsubst <<< ${LOCAL_TMP})
+##echo "'${STORAGE}'"
+##
+### make sure ${STORAGE} exists
+##mkdir -p ${STORAGE}
+##
+### obtain list of modules to be loaded
+##LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
+##echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'"
+##
+### load modules if LOAD_MODULES is not empty
+##if [[ ! -z ${LOAD_MODULES} ]]; then
+##    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+##    do
+##        echo "bot/build.sh: loading module '${mod}'"
+##        module load ${mod}
+##    done
+##else
+##    echo "bot/build.sh: no modules to be loaded"
+##fi
+
+#cpu_target_arch=$(cfg_get_value "architecture" "software_subdir" | cut -d/ -f1)
+host_arch=$(uname -m)
+eessi_arch=${cpu_target_arch:-${host_arch}}
+eessi_os=linux
+#job_version=$(cfg_get_value "repository" "repo_version")
+eessi_version=${job_version:-2023.04}
+#job_repo=$(cfg_get_value "repository" "repo_name")
+eessi_repo=${job_repo:-pilot.eessi-hpc.org}
+tar_topdir=/cvmfs/${eessi_repo}/versions
+
+# determine job output file
+job_out_file=slurm-${SLURM_JOB_ID}.out
+job_result_file=_bot_job${SLURM_JOB_ID}.result
+if [[ ! -e ${job_out_file} ]]; then
+    echo "[RESULT]" > ${job_result_file}
+    echo "summary = :thinking: UNKNOWN" >> ${job_result_file}
+    echo "details = _job output file '${job_out_file}' not found/not accessible_" >> ${job_result_file}
+    exit 0
+fi
+
+# status of build job (SUCCESS/FAILURE) + details
+# SUCCESS (all of)
+# - last line with failed=0
+# - tarball
+# FAILED (one of)
+# - no last line with failed=0
+# - no tarball
+
+play_recap=0
+PLAY_RECAP=$(grep -A1 "PLAY RECAP" ${job_out_file})
+ec=$?
+echo "PLAY_RECAP.ec=${ec}"
+[[ ${ec} -eq 0 ]] && play_recap=0 || play_recap=1
+echo "play_recap=${play_recap}"
+
+found_line_with_failed=0
+echo "${PLAY_RECAP}" | grep "failed=" > /dev/null
+ec=$?
+echo "FAILED=.ec=${ec}"
+[[ ${ec} -eq 0 ]] && found_line_with_failed=0 || found_line_with_failed=1
+echo "found_line_with_failed=${found_line_with_failed}"
+
+failed_eq_zero=0
+echo "${PLAY_RECAP}" | grep "failed=0" > /dev/null
+ec=$?
+echo "FAILED=0.ec=${ec}"
+[[ ${ec} -eq 0 ]] && failed_eq_zero=0 || failed_eq_zero=1
+echo "failed_eq_zero=${failed_eq_zero}"
+
+found_tarballs=0
+tarballs=$(ls eessi-${eessi_version}-compat-linux-${eessi_arch}-*.tar.gz 2>&1)
+ec=$?
+echo "TARBALLS.ec=${ec}"
+if [[ ${ec} -eq 0 ]]; then
+    found_tarballs=0
+else
+    found_tarballs=1
+fi
+echo "found_tarballs=${found_tarballs}"
+
+if [[ ${failed_eq_zero} -eq 0 ]] && [[ ${found_tarballs} -eq 0 ]]; then
+    # SUCCESS
+    echo "[RESULT]" > ${job_result_file}
+    echo "summary = :grin: SUCCESS" >> ${job_result_file}
+    echo "details =" >> ${job_result_file}
+    echo "    no task failed" >> ${job_result_file}
+    echo "    found tarball(s)" >> ${job_result_file}
+    echo "artefacts =" >> ${job_result_file}
+    echo "${tarballs}" | sed -e 's/^/    /' >> ${job_result_file}
+    exit 0
+else
+    # FAILURE
+    echo "[RESULT]" > ${job_result_file}
+    echo "summary = :cry: FAILURE" >> ${job_result_file}
+    echo "details =" >> ${job_result_file}
+    if [[ ${failed_eq_zero} -eq 0 ]]; then
+        echo "    no task failed" >> ${job_result_file}
+    else
+        echo "    some task(s) failed" >> ${job_result_file}
+    fi
+    if [[ ${found_tarballs} -eq 0 ]]; then
+        echo "    found tarball(s)" >> ${job_result_file}
+    else
+        echo "    no tarball found" >> ${job_result_file}
+    fi
+    echo "artefacts =" >> ${job_result_file}
+    if [[ ${found_tarballs} -eq 0 ]]; then
+        echo "${tarballs}" | sed -e 's/^/    /' >> ${job_result_file}
+    fi
+    exit 0
+fi

From 979b5f408e9b0e195c4f55f019f28db57e5a22ff Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 10 Apr 2023 21:41:00 +0200
Subject: [PATCH 24/25] cleanup after reverting to using portage

---
 .../compatibility_layer/defaults/main.yml     |  2 +-
 .../compatibility_layer/tasks/add_overlay.yml | 10 ----------
 .../compatibility_layer/tasks/cleanup.yml     | 19 -------------------
 .../tasks/install_packages.yml                | 13 -------------
 .../tasks/set_glibc_trusted_dirs.yml          |  7 -------
 5 files changed, 1 insertion(+), 50 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
index c87e83c7..c3effec8 100644
--- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
+++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
@@ -5,7 +5,7 @@ eessi_version: "2023.04"
 custom_overlays:
   - name: eessi
     source: git
-    url: https://github.com/trz42/gentoo-overlay.git
+    url: https://github.com/EESSI/gentoo-overlay.git
     eclass-overrides: true
 
 cvmfs_repository: pilot.eessi-hpc.org
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
index 259785ed..a6d5d12e 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml
@@ -11,9 +11,6 @@
     state: present
   vars:
     ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
-#  ansible.builtin.command:
-#    cmd: emerge app-eselect/eselect-repository
-#    creates: "{{ gentoo_prefix_path }}/etc/eselect/repository.conf"
 
 # We need git in order to add Gentoo overlays hosted on git repositories.
 - name: Install git
@@ -22,9 +19,6 @@
     state: present
   vars:
     ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
-#  ansible.builtin.command:
-#    cmd: emerge dev-vcs/git
-#    creates: "{{ gentoo_prefix_path }}/usr/bin/git"
 
 - name: Check which repositories have been installed
   ansible.builtin.command: eselect repository list -i
@@ -54,10 +48,6 @@
     verbose: true
   vars:
     ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
-#  ansible.builtin.command:
-#    cmd: emerge --sync
-#  register: sync_repos
-#  changed_when: sync_repos.rc != 0
 
 - name: Find all files and directories in the etc/portage directory of the overlay
   ansible.builtin.find:
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
index 73cceb91..3f3154c9 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml
@@ -1,19 +1,5 @@
 # Clean up
 ---
-#- name: Install redundant packages
-#  ansible.builtin.command:
-#    cmd: emerge --verbose "{{ item }}"
-#  register: emerge_redundant
-#  with_items: "{{ prefix_remove_packages }}"
-#  changed_when: emerge_redundant.rc != 0
-#
-#- name: Remove redundant packages from world set
-#  ansible.builtin.command:
-#    cmd: emerge --deselect --verbose "{{ item }}"
-#  register: deselect_redundant
-#  with_items: "{{ prefix_remove_packages }}"
-#  changed_when: deselect_redundant.rc != 0
-
 - name: Remove redundant packages
   community.general.portage:
     package: "{{ item }}"
@@ -21,8 +7,3 @@
   with_items: "{{ prefix_remove_packages }}"
   vars:
     ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
-#  ansible.builtin.command:
-#    cmd: emerge --unmerge --keep-going=y --verbose "{{ item }}"
-#  register: unmerge_redundant
-#  with_items: "{{ prefix_remove_packages }}"
-#  changed_when: unmerge_redundant.rc != 0
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
index df5f05b1..b006a17e 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml
@@ -9,13 +9,6 @@
     ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
   tags:
     - set
-#  ansible.builtin.command:
-#    cmd: emerge --noreplace "@{{ item }}"
-#  register: emerge_item_from_set
-#  with_items: "{{ package_sets }}"
-#  tags:
-#    - set
-#  changed_when: emerge_item_from_set.rc != 0
 
 - name: Install additional packages
   community.general.portage:
@@ -24,9 +17,3 @@
   with_items: "{{ prefix_packages }}"
   vars:
     ansible_python_interpreter: "{{ gentoo_prefix_path }}/usr/bin/python3"
-#  ansible.builtin.command:
-#    cmd: emerge "{{ item }}"
-#  register: emerge_additional_item
-#  with_items: "{{ prefix_packages }}"
-#  when: prefix_packages is iterable
-#  changed_when: emerge_additional_item.rc != 0
diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
index 8710960b..ac587f97 100644
--- a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
+++ b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml
@@ -19,13 +19,6 @@
   when:
     - eessi_host_os == "linux"
     - glibc_extra_emake.stdout != "user-defined-trusted-dirs=" + ":".join(prefix_user_defined_trusted_dirs)
-#  ansible.builtin.command:
-#    cmd: emerge --oneshot sys-libs/glibc
-#  environment:
-#    EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}"
-#  when:
-#    - eessi_host_os == "linux"
-#    - glibc_extra_emake.stdout != "user-defined-trusted-dirs=" + ":".join(prefix_user_defined_trusted_dirs)
 
 - name: Create portage env directory
   ansible.builtin.file:

From dbb1a1946826ccb44b7f531f97de1645a642a322 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Mon, 10 Apr 2023 21:54:52 +0200
Subject: [PATCH 25/25] cleanup default settings

---
 .../roles/compatibility_layer/defaults/main.yml | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
index c3effec8..122d1b49 100644
--- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
+++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml
@@ -22,10 +22,13 @@ gentoo_git_repo: https://github.com/gentoo/gentoo.git
 # gentoo_git_commit: b9ebfbaef10f4740671c8bd90c6d2c929f8379bc
 # April 5 (8c0ce1bcd089d5badcb39afce98111bf4cfbc8f0)
 # gentoo_git_commit: 8c0ce1bcd089d5badcb39afce98111bf4cfbc8f0
-# March 13 (7db97f5a3fc7ca8eced2a94e5f9481f7556cf908) one commit before 3a38f9b245e674722d5d58dfbb6baafca2b26975 (which introduced nghttp2 1.52.0 which switched to using cmake)
-# gentoo_git_commit: 7db97f5a3fc7ca8eced2a94e5f9481f7556cf908
-# March 13 (3a38f9b245e674722d5d58dfbb6baafca2b26975) commit which introduced nghttp2 1.52.0 which switched to using cmake
-gentoo_git_commit: 3a38f9b245e674722d5d58dfbb6baafca2b26975
+# March 13 (3a38f9b245e674722d5d58dfbb6baafca2b26975) commit which introduced
+# version 1.52.0 of nghttp2 (this commit switched to using cmake and likely
+# caused a circular dependency)
+# gentoo_git_commit: 3a38f9b245e674722d5d58dfbb6baafca2b26975
+# March 13 (7db97f5a3fc7ca8eced2a94e5f9481f7556cf908) one commit before
+# 3a38f9b245e674722d5d58dfbb6baafca2b26975
+gentoo_git_commit: 7db97f5a3fc7ca8eced2a94e5f9481f7556cf908
 prefix_required_space: 15 GB
 prefix_default_gcc: 9.5.0
 prefix_user_defined_trusted_dirs:
@@ -34,12 +37,6 @@ prefix_mask_packages: |
   # stick to GCC 9.x; using a too recent compiler in the compat layer complicates stuff in the software layer,
   # see for example https://github.com/EESSI/software-layer/issues/151
   >=sys-devel/gcc-10
-  # it looks like sys-apps/portaga-3.0.45.3-r2 creates a circular dependency that wasn't there before (version 3.0.45.2 with commit from March 7)
-  # didn't solve the problem with circular dependencies
-  # >=sys-apps/portage-3.0.45.3  # it looks like sys-apps/portaga-3.0.45.3-r2 creates a circular dependency that wasn't there before (version 3.0.45.2 with commit from March 7)
-  # net-libs/nghttp2-1.52.0 was introduced on March 13 (3a38f9b245e674722d5d58dfbb6baafca2b26975) switching to cmake ...
-  # .. instead of using a commit before (7db97f5a3fc7ca8eced2a94e5f9481f7556cf908) here we try to mask any version equal or younger
-  # >= net-libs/nghttp2-1.52.0
 prefix_unmask_packages: |
   # unmask older GCC to make it installable
   =sys-devel/gcc-9*