From 5abf2ca5f75feced62a4f6d42eb2f219fcfd431e Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 14:26:43 +0200
Subject: [PATCH 01/11] sync EESSI/NESSI 2023.06 - round 1

---
 .github/workflows/test_eessi.yml |  22 +--
 EESSI-pilot-install-software.sh  | 293 ++++---------------------------
 bot/check-result.sh              | 210 ++++++++++++++++++++++
 check_missing_installations.sh   |   4 +
 configure_easybuild              |   1 -
 load_easybuild_module.sh         |   4 +-
 6 files changed, 261 insertions(+), 273 deletions(-)
 create mode 100755 bot/check-result.sh

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 04195dd619..5e496a6b7d 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -31,17 +31,17 @@ jobs:
               cvmfs_http_proxy: DIRECT
               cvmfs_repositories: pilot.eessi-hpc.org
 
-        - name: Test check_missing_installations.sh script
-          run: |
-              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
-              module load EasyBuild
-              eb --version
-              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
-              export EESSI_OS_TYPE=linux
-              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
-              env | grep ^EESSI | sort
-              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
-              ./check_missing_installations.sh
+#        - name: Test check_missing_installations.sh script
+#          run: |
+#              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+#              module load EasyBuild
+#              eb --version
+#              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+#              export EESSI_OS_TYPE=linux
+#              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+#              env | grep ^EESSI | sort
+#              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
+#              ./check_missing_installations.sh
 
         - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index d9bcf20231..88194e0778 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -50,28 +50,28 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))
 
-source $TOPDIR/scripts/utils.sh
+source ${TOPDIR}/scripts/utils.sh
 
-# honor $TMPDIR if it is already defined, use /tmp otherwise
-if [ -z $TMPDIR ]; then
-    export WORKDIR=/tmp/$USER
+# honor ${TMPDIR} if it is already defined, use /tmp otherwise
+if [ -z ${TMPDIR} ]; then
+    export WORKDIR=/tmp/${USER}
 else
-    export WORKDIR=$TMPDIR/$USER
+    export WORKDIR=${TMPDIR}/${USER}
 fi
 
 TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
 
-source $TOPDIR/init/minimal_eessi_env
+source ${TOPDIR}/init/minimal_eessi_env
 
-if [ -d $EESSI_CVMFS_REPO ]; then
-    echo_green "$EESSI_CVMFS_REPO available, OK!"
+if [ -d ${EESSI_CVMFS_REPO} ]; then
+    echo_green "${EESSI_CVMFS_REPO} available, OK!"
 else
-    fatal_error "$EESSI_CVMFS_REPO is not available!"
+    fatal_error "${EESSI_CVMFS_REPO} is not available!"
 fi
 
-# make sure we're in Prefix environment by checking $SHELL
+# make sure we're in Prefix environment by checking ${SHELL}
 if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
     echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
 else
@@ -79,30 +79,30 @@ else
 fi
 
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+export PYTHONPYCACHEPREFIX=${TMPDIR}/pycache
 
 DETECTION_PARAMETERS=''
 GENERIC=0
 EB='eb'
-if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
+if [[ "${EASYBUILD_OPTARCH}" == "GENERIC" ]]; then
     echo_yellow ">> GENERIC build requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
+    DETECTION_PARAMETERS="${DETECTION_PARAMETERS} --generic"
     GENERIC=1
     EB='eb --optarch=GENERIC'
 fi
 
 echo ">> Determining software subdirectory to use for current build host..."
-if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+if [ -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 ${TOPDIR}/eessi_software_subdir.py ${DETECTION_PARAMETERS})
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py ${DETECTION_PARAMETERS}' script"
 else
   echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
 fi
 
-# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+# Set all the EESSI environment variables (respecting ${EESSI_SOFTWARE_SUBDIR_OVERRIDE})
 # $EESSI_SILENT - don't print any messages
 # $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source ${TOPDIR}/init/eessi_environment_variables
 
 if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
     fatal_error "Failed to determine software subdirectory?!"
@@ -113,9 +113,9 @@ else
 fi
 
 echo ">> Initializing Lmod..."
-source $EPREFIX/usr/share/Lmod/init/bash
-ml_version_out=$TMPDIR/ml.out
-ml --version &> $ml_version_out
+source ${EPREFIX}/usr/share/Lmod/init/bash
+ml_version_out=${TMPDIR}/ml.out
+ml --version &> ${ml_version_out}
 if [[ $? -eq 0 ]]; then
     echo_green ">> Found Lmod ${LMOD_VERSION}"
 else
@@ -123,262 +123,37 @@ else
 fi
 
 echo ">> Configuring EasyBuild..."
-source $TOPDIR/configure_easybuild
+source ${TOPDIR}/configure_easybuild
 
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge
 # ignore current $MODULEPATH entirely
-module unuse $MODULEPATH
-module use $EASYBUILD_INSTALLPATH/modules/all
+module unuse ${MODULEPATH}
+module use ${EASYBUILD_INSTALLPATH}/modules/all
 if [[ -z ${MODULEPATH} ]]; then
     fatal_error "Failed to set up \$MODULEPATH?!"
 else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-REQ_EB_VERSION='4.5.0'
+for eb_version in '4.7.2'; do
+    # load EasyBuild module (will be installed if it's not available yet)
+    source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
-# load EasyBuild module (will be installed if it's not available yet)
-source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION}
+    echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
-echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."
-
-# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up,
-# see https://github.com/EESSI/software-layer/issues/123
-# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557
-ok_msg="Java installed, off to a good (?) start!"
-fail_msg="Failed to install Java, woopsie..."
-$EB Java-11.eb --robot --include-easyblocks-from-pr 2557
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# install GCC for foss/2020a
-export GCC_EC="GCC-9.3.0.eb"
-echo ">> Starting slow with ${GCC_EC}..."
-ok_msg="${GCC_EC} installed, yippy! Off to a good start..."
-fail_msg="Installation of ${GCC_EC} failed!"
-# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453,
-# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place
-$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# install CMake with custom easyblock that patches CMake when --sysroot is used
-echo ">> Install CMake with fixed easyblock to take into account --sysroot"
-ok_msg="CMake installed!"
-fail_msg="Installation of CMake failed, what the ..."
-$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946
-echo ">> Installing OpenBLAS..."
-ok_msg="Done with OpenBLAS!"
-fail_msg="Installation of OpenBLAS failed!"
-if [[ $GENERIC -eq 1 ]]; then
-    echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS."
-    openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946"
-else
-    openblas_include_easyblocks_from_pr=''
-fi
-$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OpenMPI..."
-ok_msg="OpenMPI installed, w00!"
-fail_msg="Installation of OpenMPI failed, that's not good..."
-$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# install Python
-echo ">> Install Python 2.7.18 and Python 3.8.2..."
-ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!"
-fail_msg="Installation of Python failed, oh no..."
-$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Perl..."
-ok_msg="Perl installed, making progress..."
-fail_msg="Installation of Perl failed, this never happens..."
-# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
-# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
-$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Qt5..."
-ok_msg="Qt5 installed, phieuw, that was a big one!"
-fail_msg="Installation of Qt5 failed, that's frustrating..."
-$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# skip test step when installing SciPy-bundle on aarch64,
-# to dance around problem with broken numpy tests;
-# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959
-echo ">> Installing SciPy-bundle"
-ok_msg="SciPy-bundle installed, yihaa!"
-fail_msg="SciPy-bundle installation failed, bummer..."
-SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb
-if [[ "$(uname -m)" == "aarch64" ]]; then
-  $EB $SCIPY_EC --robot --skip-test-step
-else
-  $EB $SCIPY_EC --robot
-fi
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing GROMACS..."
-ok_msg="GROMACS installed, wow!"
-fail_msg="Installation of GROMACS failed, damned..."
-$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)!
-# 32GB is sufficient to build with 16 cores
-echo ">> Installing OpenFOAM (twice!)..."
-ok_msg="OpenFOAM installed, now we're talking!"
-fail_msg="Installation of OpenFOAM failed, we were so close..."
-$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then
-    echo ">> Installing QuantumESPRESSO..."
-    ok_msg="QuantumESPRESSO installed, let's go quantum!"
-    fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!"
-    $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot
-    check_exit_code $? "${ok_msg}" "${fail_msg}"
-fi
-
-echo ">> Installing R 4.0.0 (better be patient)..."
-ok_msg="R installed, wow!"
-fail_msg="Installation of R failed, so sad..."
-$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Bioconductor 3.11 bundle..."
-ok_msg="Bioconductor installed, enjoy!"
-fail_msg="Installation of Bioconductor failed, that's annoying..."
-$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing TensorFlow 2.3.1..."
-ok_msg="TensorFlow 2.3.1 installed, w00!"
-fail_msg="Installation of TensorFlow failed, why am I not surprised..."
-$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Horovod 0.21.3..."
-ok_msg="Horovod installed! Go do some parallel training!"
-fail_msg="Horovod installation failed. There comes the headache..."
-$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then
-
-    echo ">> Installing code-server 3.7.3..."
-    ok_msg="code-server 3.7.3 installed, now you can use VS Code!"
-    fail_msg="Installation of code-server failed, that's going to be hard to fix..."
-    $EB code-server-3.7.3.eb --robot
-    check_exit_code $? "${ok_msg}" "${fail_msg}"
-fi
-
-echo ">> Installing RStudio-Server 1.3.1093..."
-ok_msg="RStudio-Server installed, enjoy!"
-fail_msg="Installation of RStudio-Server failed, might be OS deps..."
-$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OSU-Micro-Benchmarks 5.6.3..."
-ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
-fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
-$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Spark 3.1.1..."
-ok_msg="Spark installed, set off the fireworks!"
-fail_msg="Installation of Spark failed, no fireworks this time..."
-$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing IPython 7.15.0..."
-ok_msg="IPython installed, launch your Jupyter Notebooks!"
-fail_msg="Installation of IPython failed, that's unexpected..."
-$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing WRF 3.9.1.1..."
-ok_msg="WRF installed, it's getting hot in here!"
-fail_msg="Installation of WRF failed, that's unexpected..."
-OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing R 4.1.0 (better be patient)..."
-ok_msg="R installed, wow!"
-fail_msg="Installation of R failed, so sad..."
-$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Nextflow 22.10.1..."
-ok_msg="Nextflow installed, the work must flow..."
-fail_msg="Installation of Nextflow failed, that's unexpected..."
-$EB -r --from-pr 16531 Nextflow-22.10.1.eb
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..."
-ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
-fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
-$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing EasyBuild 4.5.1..."
-ok_msg="EasyBuild v4.5.1 installed"
-fail_msg="EasyBuild v4.5.1 failed to install"
-$EB --from-pr 14545 --include-easyblocks-from-pr 2805
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1
-check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1"
-
-echo ">> Installing SciPy-bundle with foss/2021a..."
-ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age"
-fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..."
-# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454
-# which includes patch to fix installation with recent Linux kernel headers
-$EB --from-pr 14454 GCCcore-10.3.0.eb --robot
-# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
-# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
-$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640
-# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set
-# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248
-$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
-# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584
-# that includes patch to fix bootstrap problem when using alternate sysroot
-$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
-# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
-# which includes a patch to fix installation on POWER
-$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
-# ignore failing FlexiBLAS tests when building on POWER;
-# some tests are failing due to a segmentation fault due to "invalid memory reference",
-# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476;
-# using -fstack-protector-strong -fstack-clash-protection should fix that,
-# but it doesn't for some reason when building for ppc64le/generic...
-if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then
-    $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure
-else
-    $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb
-fi
-
-$EB SciPy-bundle-2021.05-foss-2021a.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-### add packages here
+    ${TOPDIR}/check_missing_installations.sh
+done
 
 echo ">> Creating/updating Lmod cache..."
 export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"
-if [ ! -f $LMOD_RC ]; then
-    python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
-    check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC"
+if [ ! -f ${LMOD_RC} ]; then
+    python3 ${TOPDIR}/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
+    check_exit_code $? "${LMOD_RC} created" "Failed to create ${LMOD_RC}"
 fi
 
-$TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
-
-$TOPDIR/check_missing_installations.sh
+${TOPDIR}/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
diff --git a/bot/check-result.sh b/bot/check-result.sh
new file mode 100755
index 0000000000..e28ae4e38e
--- /dev/null
+++ b/bot/check-result.sh
@@ -0,0 +1,210 @@
+#!/bin/bash
+#
+# Script to check the result of building the EESSI software layer.
+# Intended use is that it is called by a (batch) job running on a compute
+# node.
+#
+# This script is part of the EESSI compatibility layer, see
+# https://github.com/EESSI/compatibility-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# result cases
+
+#  - SUCCESS (all of)
+#    - working directory contains slurm-JOBID.out file
+#    - working directory contains eessi*tar.gz
+#    - no message ERROR
+#    - no message FAILED
+#    - no message ' required modules missing:'
+#    - one or more of 'No missing modules!'
+#    - message regarding created tarball
+#  - FAILED (one of ... implemented as NOT SUCCESS)
+#    - no slurm-JOBID.out file
+#    - no tarball
+#    - message with ERROR
+#    - message with FAILED
+#    - message with ' required modules missing:'
+#    - no message regarding created tarball
+
+# stop as soon as something fails
+# set -e
+
+TOPDIR=$(dirname $(realpath $0))
+
+source ${TOPDIR}/../scripts/utils.sh
+source ${TOPDIR}/../scripts/cfg_files.sh
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo " OPTIONS:"
+  echo "  -h | --help    - display this usage information [default: false]"
+  echo "  -v | --verbose - display more information [default: false]"
+}
+
+# set defaults for command line arguments
+VERBOSE=0
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -h|--help)
+      display_help
+      exit 0
+      ;;
+    -v|--verbose)
+      VERBOSE=1
+      shift 1
+      ;;
+    --)
+      shift
+      POSITIONAL_ARGS+=("$@") # save positional args
+      break
+      ;;
+    -*|--*)
+      fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+job_dir=${PWD}
+
+[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
+
+GP_slurm_out="slurm-${SLURM_JOB_ID}.out"
+job_out=$(ls ${job_dir} | grep "${GP_slurm_out}")
+[[ $? -eq 0 ]] && SLURM=1 || SLURM=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
+
+GP_error='ERROR: '
+grep_out=$(grep "${GP_error}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_failed='FAILED: '
+grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_req_missing=' required modules missing:'
+grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && MISSING=1 || MISSING=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_no_missing='No missing modules!'
+grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_tgz_created="tar.gz created!"
+TARBALL=
+grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out})
+if [[ $? -eq 0 ]]; then
+    TGZ=1
+    TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@')
+else
+    TGZ=0
+fi
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}"
+[[ ${VERBOSE} -ne 0 ]] && echo "  test name  : result (expected result)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+[[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+
+job_result_file=_bot_job${SLURM_JOB_ID}.result
+
+if [[ ${SLURM} -eq 1 ]] && \
+   [[ ${ERROR} -eq 0 ]] && \
+   [[ ${FAILED} -eq 0 ]] && \
+   [[ ${MISSING} -eq 0 ]] && \
+   [[ ${NO_MISSING} -eq 1 ]] && \
+   [[ ${TGZ} -eq 1 ]] && \
+   [[ ! -z ${TARBALL} ]]; then
+    # SUCCESS
+    echo "[RESULT]" > ${job_result_file}
+    echo "summary = :grin: SUCCESS" >> ${job_result_file}
+    echo "details =" >> ${job_result_file}
+else
+    # FAILURE
+    echo "[RESULT]" > ${job_result_file}
+    echo "summary = :cry: FAILURE" >> ${job_result_file}
+    echo "details =" >> ${job_result_file}
+fi
+
+function succeeded() {
+    echo "    :heavy_check_mark: ${1}"
+}
+
+function failed() {
+    echo "    :heavy_multiplication_x: ${1}"
+}
+
+if [[ ${SLURM} -eq 1 ]]; then
+    succeeded "job output file <code>${job_out}</code>" >> ${job_result_file}
+else
+    failed "no job output file matching <code>${GP_slurm_out}</code>" >> ${job_result_file}
+fi
+
+if [[ ${ERROR} -eq 0 ]]; then
+    succeeded "no message matching <code>${GP_error}</code>" >> ${job_result_file}
+else
+    failed "found message matching <code>${GP_error}</code>" >> ${job_result_file}
+fi
+
+if [[ ${FAILED} -eq 0 ]]; then
+    succeeded "no message matching <code>${GP_failed}</code>" >> ${job_result_file}
+else
+    failed "found message matching <code>${GP_failed}</code>" >> ${job_result_file}
+fi
+
+if [[ ${MISSING} -eq 0 ]]; then
+    succeeded "no message matching <code>${GP_req_missing}</code>" >> ${job_result_file}
+else
+    failed "found message matching <code>${GP_req_missing}</code>" >> ${job_result_file}
+fi
+
+if [[ ${NO_MISSING} -eq 1 ]]; then
+    succeeded "found message(s) matching <code>${GP_no_missing}</code>" >> ${job_result_file}
+else
+    failed "no message matching <code>${GP_no_missing}</code>" >> ${job_result_file}
+fi
+
+if [[ ${TGZ} -eq 1 ]]; then
+    succeeded "found message matching <code>${GP_tgz_created}</code>" >> ${job_result_file}
+else
+    failed "no message matching <code>${GP_tgz_created}</code>" >> ${job_result_file}
+fi
+
+echo "artefacts =" >> ${job_result_file}
+
+if [[ ! -z ${TARBALL} ]]; then
+    echo "    ${TARBALL}" >> ${job_result_file}
+fi
+
+exit 0
diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 4a5316c09f..945bf32b19 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -25,6 +25,10 @@ echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
 # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
 # PR #16531: Nextflow-22.10.1.eb
+# ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
+# PR 16531 not needed since we use EB v4.7.0
+# this, however, breaks the GHA https://github.com/NorESSI/software-layer/blob/main/.github/workflows/test_eessi.yml
+# because it uses the EESSI pilot which only provides EB 4.5.1, so adding it back
 ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
 exit_code=${PIPESTATUS[0]}
 
diff --git a/configure_easybuild b/configure_easybuild
index 245553f342..19b2d7454b 100644
--- a/configure_easybuild
+++ b/configure_easybuild
@@ -13,7 +13,6 @@ export EASYBUILD_ZIP_LOGS=bzip2
 
 export EASYBUILD_RPATH=1
 export EASYBUILD_FILTER_ENV_VARS=LD_LIBRARY_PATH
-export EASYBUILD_READ_ONLY_INSTALLDIR=1
 
 # assume that eb_hooks.py is located in same directory as this script (configure_easybuild)
 TOPDIR=$(dirname $(realpath $BASH_SOURCE))
diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh
index 4ff2a3c37c..9897059f82 100755
--- a/load_easybuild_module.sh
+++ b/load_easybuild_module.sh
@@ -23,14 +23,14 @@ fi
 EB_VERSION=${1}
 
 # make sure that environment variables that we expect to be set are indeed set
-if [ -z "${TMPDIR}" ]; then 
+if [ -z "${TMPDIR}" ]; then
     echo "\$TMPDIR is not set" >&2
     exit 2
 fi
 
 # ${EB} is used to specify which 'eb' command should be used;
 # can potentially be more than just 'eb', for example when using 'eb --optarch=GENERIC'
-if [ -z "${EB}" ]; then 
+if [ -z "${EB}" ]; then
     echo "\$EB is not set" >&2
     exit 2
 fi

From 8abf2c2d0d2f6900a20af7b41b23be899ef16611 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 16:52:37 +0200
Subject: [PATCH 02/11] prepare installation of software + rename old easystack
 file

---
 EESSI-pilot-install-software.sh  | 15 ++++++++++++++-
 check_missing_installations.sh   | 15 +++++----------
 eessi-2023.06-eb-4.7.2-2021a.yml |  1 +
 3 files changed, 20 insertions(+), 11 deletions(-)
 create mode 100644 eessi-2023.06-eb-4.7.2-2021a.yml

diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 88194e0778..8427ebe374 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -143,7 +143,20 @@ for eb_version in '4.7.2'; do
 
     echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
-    ${TOPDIR}/check_missing_installations.sh
+    for gen in '2021a'; do
+
+        es="eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-${gen}.yml"
+
+        if [ -f ${es} ]; then
+            echo_green "Feeding easystack file ${es} to EasyBuild..."
+
+            ${EB} --easystack ${TOPDIR}/${es} --missing && ${EB} --easystack ${TOPDIR}/${es} --robot
+
+            $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es}
+        else
+            fatal_error "Easystack file ${es} not found!"
+        fi
+    done
 done
 
 echo ">> Creating/updating Lmod cache..."
diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 945bf32b19..3627d1d0b5 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to check for missing installations in EESSI pilot software stack (version 2021.12)
+# Script to check for missing installations in EESSI pilot software stack (version 2023.06)
 #
 # author: Kenneth Hoste (@boegel)
 # author: Thomas Roeblitz (@trz42)
@@ -10,10 +10,11 @@
 
 TOPDIR=$(dirname $(realpath $0))
 
-if [ -z ${EESSI_PILOT_VERSION} ]; then
-    echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2
+if [ $# -ne 1 ]; then
+    echo "ERROR: Usage: $0 <path to easystack file>" >&2
     exit 1
 fi
+easystack=$1
 
 LOCAL_TMPDIR=$(mktemp -d)
 
@@ -23,13 +24,7 @@ source $TOPDIR/configure_easybuild
 
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
-# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
-# PR #16531: Nextflow-22.10.1.eb
-# ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
-# PR 16531 not needed since we use EB v4.7.0
-# this, however, breaks the GHA https://github.com/NorESSI/software-layer/blob/main/.github/workflows/test_eessi.yml
-# because it uses the EESSI pilot which only provides EB 4.5.1, so adding it back
-${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
+${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out}
 exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml
new file mode 100644
index 0000000000..2add7fb98b
--- /dev/null
+++ b/eessi-2023.06-eb-4.7.2-2021a.yml
@@ -0,0 +1 @@
+easyconfigs:

From c58d1764a78615a2aa54d12eaab43e6c56e01d1d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 16:54:29 +0200
Subject: [PATCH 03/11] always use --experimental

---
 configure_easybuild | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/configure_easybuild b/configure_easybuild
index 19b2d7454b..8e79f3a35a 100644
--- a/configure_easybuild
+++ b/configure_easybuild
@@ -35,3 +35,6 @@ fi
 export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER
 
 export EASYBUILD_MODULE_EXTENSIONS=1
+
+# need to enable use of experimental features, since we're using easystack files
+export EASYBUILD_EXPERIMENTAL=1

From b9ca155c3da4dc17c0ec5f37109de4811e3321b4 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 17:18:00 +0200
Subject: [PATCH 04/11] update eb hooks

---
 eb_hooks.py | 63 ++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 43 insertions(+), 20 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 2fba925b01..29ec61c324 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -11,6 +11,14 @@
 from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
 from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
 
+# prefer importing LooseVersion from easybuild.tools, but fall back to distutils
+# in case EasyBuild <= 4.7.0 is used
+try:
+    from easybuild.tools import LooseVersion
+except ImportError:
+    from distutils.version import LooseVersion
+
+
 EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
 
 
@@ -56,11 +64,11 @@ def parse_hook(ec, *args, **kwargs):
         PARSE_HOOKS[ec.name](ec, eprefix)
 
 
-def pre_configure_hook(self, *args, **kwargs):
-    """Main pre-configure hook: trigger custom functions based on software name."""
-
-    if self.name in PRE_CONFIGURE_HOOKS:
-        PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
+#def pre_configure_hook(self, *args, **kwargs):
+#    """Main pre-configure hook: trigger custom functions based on software name."""
+#
+#    if self.name in PRE_CONFIGURE_HOOKS:
+#        PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
 
 
 def pre_prepare_hook(self, *args, **kwargs):
@@ -91,10 +99,11 @@ def pre_prepare_hook(self, *args, **kwargs):
                   mpi_family, rpath_override_dirs)
 
 
-def gcc_postprepare(self, *args, **kwargs):
+def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs):
     """
     Post-configure hook for GCCcore:
-    - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu'
+    - copy RPATH wrapper script for linker commands to also have a wrapper in
+      place with system type prefix like 'x86_64-pc-linux-gnu'
     """
     if self.name == 'GCCcore':
         config_guess = obtain_config_guess()
@@ -121,6 +130,7 @@ def gcc_postprepare(self, *args, **kwargs):
     else:
         raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!")
 
+
 def post_prepare_hook(self, *args, **kwargs):
     """Main post-prepare hook: trigger custom functions."""
 
@@ -134,7 +144,7 @@ def post_prepare_hook(self, *args, **kwargs):
         POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
 
 
-def cgal_toolchainopts_precise(ec, eprefix):
+def parse_hook_cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
     if ec.name == 'CGAL':
         if get_cpu_architecture() == POWER:
@@ -147,7 +157,7 @@ def cgal_toolchainopts_precise(ec, eprefix):
         raise EasyBuildError("CGAL-specific hook triggered for non-CGAL easyconfig?!")
 
 
-def fontconfig_add_fonts(ec, eprefix):
+def parse_hook_fontconfig_add_fonts(ec, eprefix):
     """Inject --with-add-fonts configure option for fontconfig."""
     if ec.name == 'fontconfig':
         # make fontconfig aware of fonts included with compat layer
@@ -158,7 +168,7 @@ def fontconfig_add_fonts(ec, eprefix):
         raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!")
 
 
-def ucx_eprefix(ec, eprefix):
+def parse_hook_ucx_eprefix(ec, eprefix):
     """Make UCX aware of compatibility layer via additional configuration options."""
     if ec.name == 'UCX':
         ec.update('configopts', '--with-sysroot=%s' % eprefix)
@@ -174,7 +184,19 @@ def pre_configure_hook(self, *args, **kwargs):
         PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
 
 
-def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs):
+def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs):
+    """
+    Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC
+    """
+    if self.name == 'OpenBLAS':
+        if build_option('optarch') == OPTARCH_GENERIC:
+            for step in ('build', 'test', 'install'):
+                self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1")
+    else:
+        raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!")
+
+
+def pre_configure_hook_libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs):
     """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64."""
     if self.name == 'libfabric':
         if get_cpu_architecture() == X86_64:
@@ -187,7 +209,7 @@ def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs):
         raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!")
 
 
-def metabat_preconfigure(self, *args, **kwargs):
+def pre_configure_hook_metabat_filtered_zlib_dep(self, *args, **kwargs):
     """
     Pre-configure hook for MetaBAT:
     - take into account that zlib is a filtered dependency,
@@ -201,7 +223,7 @@ def metabat_preconfigure(self, *args, **kwargs):
         raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!")
 
 
-def wrf_preconfigure(self, *args, **kwargs):
+def pre_configure_hook_wrf_aarch64(self, *args, **kwargs):
     """
     Pre-configure hook for WRF:
     - patch arch/configure_new.defaults so building WRF with foss toolchain works on aarch64
@@ -217,17 +239,18 @@ def wrf_preconfigure(self, *args, **kwargs):
 
 
 PARSE_HOOKS = {
-    'CGAL': cgal_toolchainopts_precise,
-    'fontconfig': fontconfig_add_fonts,
-    'UCX': ucx_eprefix,
+    'CGAL': parse_hook_cgal_toolchainopts_precise,
+    'fontconfig': parse_hook_fontconfig_add_fonts,
+    'UCX': parse_hook_ucx_eprefix,
 }
 
 POST_PREPARE_HOOKS = {
-    'GCCcore': gcc_postprepare,
+    'GCCcore': post_prepare_hook_gcc_prefixed_ld_rpath_wrapper,
 }
 
 PRE_CONFIGURE_HOOKS = {
-    'libfabric': libfabric_disable_psm3_x86_64_generic,
-    'MetaBAT': metabat_preconfigure,
-    'WRF': wrf_preconfigure,
+    'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic,
+    'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep,
+    'OpenBLAS': pre_configure_hook_openblas_optarch_generic,
+    'WRF': pre_configure_hook_wrf_aarch64,
 }

From 8b99626c8643d313918479743d926b56ec983e6a Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 17:24:45 +0200
Subject: [PATCH 05/11] order functions alphabetically

---
 eb_hooks.py | 45 +++++++++++++++++++--------------------------
 1 file changed, 19 insertions(+), 26 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 29ec61c324..8df5e076e0 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -54,23 +54,6 @@ def get_rpath_override_dirs(software_name):
     return rpath_injection_dirs
 
 
-def parse_hook(ec, *args, **kwargs):
-    """Main parse hook: trigger custom functions based on software name."""
-
-    # determine path to Prefix installation in compat layer via $EPREFIX
-    eprefix = get_eessi_envvar('EPREFIX')
-
-    if ec.name in PARSE_HOOKS:
-        PARSE_HOOKS[ec.name](ec, eprefix)
-
-
-#def pre_configure_hook(self, *args, **kwargs):
-#    """Main pre-configure hook: trigger custom functions based on software name."""
-#
-#    if self.name in PRE_CONFIGURE_HOOKS:
-#        PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
-
-
 def pre_prepare_hook(self, *args, **kwargs):
     """Main pre-prepare hook: trigger custom functions."""
 
@@ -99,6 +82,19 @@ def pre_prepare_hook(self, *args, **kwargs):
                   mpi_family, rpath_override_dirs)
 
 
+def post_prepare_hook(self, *args, **kwargs):
+    """Main post-prepare hook: trigger custom functions."""
+
+    if hasattr(self, EESSI_RPATH_OVERRIDE_ATTR):
+        # Reset the value of 'rpath_override_dirs' now that we are finished with it
+        update_build_option('rpath_override_dirs', getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
+        print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
+        delattr(self, EESSI_RPATH_OVERRIDE_ATTR)
+
+    if self.name in POST_PREPARE_HOOKS:
+        POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
+
+
 def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs):
     """
     Post-configure hook for GCCcore:
@@ -131,17 +127,14 @@ def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs):
         raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!")
 
 
-def post_prepare_hook(self, *args, **kwargs):
-    """Main post-prepare hook: trigger custom functions."""
+def parse_hook(ec, *args, **kwargs):
+    """Main parse hook: trigger custom functions based on software name."""
 
-    if hasattr(self, EESSI_RPATH_OVERRIDE_ATTR):
-        # Reset the value of 'rpath_override_dirs' now that we are finished with it
-        update_build_option('rpath_override_dirs', getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
-        print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
-        delattr(self, EESSI_RPATH_OVERRIDE_ATTR)
+    # determine path to Prefix installation in compat layer via $EPREFIX
+    eprefix = get_eessi_envvar('EPREFIX')
 
-    if self.name in POST_PREPARE_HOOKS:
-        POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
+    if ec.name in PARSE_HOOKS:
+        PARSE_HOOKS[ec.name](ec, eprefix)
 
 
 def parse_hook_cgal_toolchainopts_precise(ec, eprefix):

From fe290136771f4f9fbd0743befafad34abed73b45 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 17:37:21 +0200
Subject: [PATCH 06/11] adjust tests: bump version + easystack file format

---
 .github/workflows/test_eessi.yml    | 17 ++++++++---------
 .github/workflows/tests_scripts.yml |  2 +-
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 5e496a6b7d..d3906a0f9f 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-        - 2021.12
+        - 2023.06
         EESSI_SOFTWARE_SUBDIR:
         - aarch64/generic
         - aarch64/graviton2
@@ -20,6 +20,8 @@ jobs:
         - x86_64/intel/haswell
         - x86_64/intel/skylake_avx512
         - x86_64/generic
+        EASYSTACK_FILE:
+        - eessi-2023.06-eb-4.7.2-2021a.yml
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
@@ -41,7 +43,7 @@ jobs:
 #              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
 #              env | grep ^EESSI | sort
 #              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
-#              ./check_missing_installations.sh
+#              ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}
 
         - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
@@ -52,18 +54,15 @@ jobs:
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
-              echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
-              echo "  GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
-              echo "    toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
-              echo "      SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
-              echo "        versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
-              tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+              echo "modify easystack file by adding a missing package (GCC/8.3.0)"
+              echo "  - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}}
+              tail -n 5 ${{matrix.EASYSTACK_FILE}}
               # note, check_missing_installations.sh exits 1 if a package was
               # missing, which is intepreted as false (exit code based, not
               # boolean logic), hence when the script exits 0 if no package was
               # missing it is interpreted as true, thus the test did not capture
               # the missing package
-              if ./check_missing_installations.sh; then
+              if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then
                 echo "did NOT capture missing package; test FAILED"
                 exit 1
               else
diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 5c0b3893ae..8ad4928993 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -92,6 +92,6 @@ jobs:
           # since create_directory_tarballs.sh must be accessible from within build container
           cp -a * /tmp/
           cd /tmp
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06
           # check if tarballs have been produced
           ls -l *.tar.gz

From c402acd21d3feda1d89cb508898ad7cb1f643a44 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 17:39:15 +0200
Subject: [PATCH 07/11] bump version

---
 init/eessi_defaults | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init/eessi_defaults b/init/eessi_defaults
index f482cbc269..0143dc38ab 100644
--- a/init/eessi_defaults
+++ b/init/eessi_defaults
@@ -9,4 +9,4 @@
 #
 
 export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
-export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
+export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}"

From a3e696ba6ebec1f809d7505be3c6913553045e94 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 19:40:52 +0200
Subject: [PATCH 08/11] use empty list of easyconfigs at the start

---
 eessi-2023.06-eb-4.7.2-2021a.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml
index 2add7fb98b..29301d6c77 100644
--- a/eessi-2023.06-eb-4.7.2-2021a.yml
+++ b/eessi-2023.06-eb-4.7.2-2021a.yml
@@ -1 +1 @@
-easyconfigs:
+easyconfigs: []

From 5bb31a4b73b61a33f4cb2871efc07c65ba02063c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 16 Jun 2023 20:49:56 +0200
Subject: [PATCH 09/11] minor modification to capture success when list of
 easyconfigs to install was empty

---
 bot/check-result.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index e28ae4e38e..4567f4989d 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -20,7 +20,7 @@
 #    - no message ERROR
 #    - no message FAILED
 #    - no message ' required modules missing:'
-#    - one or more of 'No missing modules!'
+#    - message 'No missing installations'
 #    - message regarding created tarball
 #  - FAILED (one of ... implemented as NOT SUCCESS)
 #    - no slurm-JOBID.out file
@@ -109,7 +109,7 @@ grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out})
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
 [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 
-GP_no_missing='No missing modules!'
+GP_no_missing='No missing installations'
 grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out})
 [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
 # have to be careful to not add searched for pattern into slurm out file

From 1593a4ba19892cb007df211603762788ec956fa0 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 18 Jun 2023 03:22:34 +0200
Subject: [PATCH 10/11] add tarball details to comment

---
 bot/check-result.sh | 430 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 351 insertions(+), 79 deletions(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 4567f4989d..78c850d3b4 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -4,8 +4,8 @@
 # Intended use is that it is called by a (batch) job running on a compute
 # node.
 #
-# This script is part of the EESSI compatibility layer, see
-# https://github.com/EESSI/compatibility-layer.git
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
 #
 # author: Thomas Roeblitz (@trz42)
 #
@@ -20,7 +20,7 @@
 #    - no message ERROR
 #    - no message FAILED
 #    - no message ' required modules missing:'
-#    - message 'No missing installations'
+#    - one or more of 'No missing modules!'
 #    - message regarding created tarball
 #  - FAILED (one of ... implemented as NOT SUCCESS)
 #    - no slurm-JOBID.out file
@@ -38,6 +38,20 @@ TOPDIR=$(dirname $(realpath $0))
 source ${TOPDIR}/../scripts/utils.sh
 source ${TOPDIR}/../scripts/cfg_files.sh
 
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    echo_red "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+else
+    echo "bot/check-result.sh: showing ${JOB_CFG_FILE} from software-layer side"
+    cat ${JOB_CFG_FILE}
+
+    echo "bot/check-result.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+    cfg_load ${JOB_CFG_FILE}
+fi
+
 display_help() {
   echo "usage: $0 [OPTIONS]"
   echo " OPTIONS:"
@@ -88,46 +102,61 @@ job_out=$(ls ${job_dir} | grep "${GP_slurm_out}")
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'"
 [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 
-GP_error='ERROR: '
-grep_out=$(grep "${GP_error}" ${job_dir}/${job_out})
-[[ $? -eq 0 ]] && ERROR=1 || ERROR=0
-# have to be careful to not add searched for pattern into slurm out file
-[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
-[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+ERROR=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_error='ERROR: '
+  grep_out=$(grep "${GP_error}" ${job_dir}/${job_out})
+  [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
 
-GP_failed='FAILED: '
-grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out})
-[[ $? -eq 0 ]] && FAILED=1 || FAILED=0
-# have to be careful to not add searched for pattern into slurm out file
-[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
-[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+FAILED=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_failed='FAILED: '
+  grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out})
+  [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
 
-GP_req_missing=' required modules missing:'
-grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out})
-[[ $? -eq 0 ]] && MISSING=1 || MISSING=0
-# have to be careful to not add searched for pattern into slurm out file
-[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
-[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+MISSING=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_req_missing=' required modules missing:'
+  grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out})
+  [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
 
-GP_no_missing='No missing installations'
-grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out})
-[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
-# have to be careful to not add searched for pattern into slurm out file
-[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'"
-[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+NO_MISSING=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_no_missing='No missing installations'
+  grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out})
+  [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
 
-GP_tgz_created="tar.gz created!"
+TGZ=-1
 TARBALL=
-grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out})
-if [[ $? -eq 0 ]]; then
-    TGZ=1
-    TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@')
-else
-    TGZ=0
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_tgz_created="tar.gz created!"
+  grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out})
+  if [[ $? -eq 0 ]]; then
+      TGZ=1
+      TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@')
+  else
+      TGZ=0
+  fi
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
-# have to be careful to not add searched for pattern into slurm out file
-[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
-[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 
 [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}"
 [[ ${VERBOSE} -ne 0 ]] && echo "  test name  : result (expected result)"
@@ -147,64 +176,307 @@ if [[ ${SLURM} -eq 1 ]] && \
    [[ ${TGZ} -eq 1 ]] && \
    [[ ! -z ${TARBALL} ]]; then
     # SUCCESS
-    echo "[RESULT]" > ${job_result_file}
-    echo "summary = :grin: SUCCESS" >> ${job_result_file}
-    echo "details =" >> ${job_result_file}
+    status="SUCCESS"
+    summary=":grin: SUCCESS"
 else
     # FAILURE
-    echo "[RESULT]" > ${job_result_file}
-    echo "summary = :cry: FAILURE" >> ${job_result_file}
-    echo "details =" >> ${job_result_file}
+    status="FAILURE"
+    summary=":cry: FAILURE"
 fi
 
-function succeeded() {
-    echo "    :heavy_check_mark: ${1}"
+### Example details/descriptions
+# Note, final string must not contain any line breaks. Below examples include
+# line breaks for the sake of readability.
+# <details>
+#   <summary>:cry: FAILURE _(click triangle for detailed information)_</summary>
+#   Details:<br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: job output file <code>slurm-470503.out</code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_multiplication_x: found message matching <code>ERROR: </code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_multiplication_x: found message matching <code>FAILED: </code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_multiplication_x: found message matching <code> required modules missing:</code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: found message(s) matching <code>No missing installations</code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: found message matching <code>tar.gz created!</code><br/>
+#   Artefacts:
+#   <li><code>eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz</code></li>
+# </details>
+#
+# <details>
+#   <summary>:grin: SUCCESS _(click triangle for detailed information)_</summary>
+#   Details:<br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: job output file <code>slurm-470503.out</code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: found message matching <code>ERROR: </code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: found message matching <code>FAILED: </code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: found message matching <code> required modules missing:</code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: found message(s) matching <code>No missing installations</code><br/>
+#   &nbsp;&nbsp;&nbsp;&nbsp;:heavy_check_mark: found message matching <code>tar.gz created!</code><br/>
+#   Artefacts:
+#   <li><code>eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz</code></li>
+# </details>
+#
+# <details>
+#   <summary>:grin: SUCCESS _(click triangle for detailed information)_</summary>
+#   <dl>
+#     <dt>_Details_</dt>
+#     <dd>
+#       :heavy_check_mark: job output file <code>slurm-4682.out</code><br/>
+#       :heavy_check_mark: no message matching <code>ERROR: </code><br/>
+#       :heavy_check_mark: no message matching <code>FAILED: </code><br/>
+#       :heavy_check_mark: no message matching <code> required modules missing:</code><br/>
+#       :heavy_check_mark: found message(s) matching <code>No missing installations</code><br/>
+#       :heavy_check_mark: found message matching <code>tar.gz created!</code><br/>
+#     </dd>
+#     <dt>_Artefacts_</dt>
+#     <dd>
+#       <details>
+#         <summary><code>eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz</code></summary>
+#         size: 234 MiB (245366784 bytes)<br/>
+#         entries: 1234<br/>
+#         modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_<br/>
+#         <pre>
+#           GCC/9.3.0.lua<br/>
+#           GCC/10.3.0.lua<br/>
+#           OpenSSL/1.1.lua
+#         </pre>
+#         software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_
+#         <pre>
+#           GCC/9.3.0/<br/>
+#           CMake/3.20.1-GCCcore-10.3.0/<br/>
+#           OpenMPI/4.1.1-GCC-10.3.0/
+#         </pre>
+#         other under _2023.04/software/linux/x86_64/intel/cascadelake/_
+#         <pre>
+#           .lmod/cache/spiderT.lua<br/>
+#           .lmod/cache/spiderT.luac_5.1<br/>
+#           .lmod/cache/timestamp
+#         </pre>
+#       </details>
+#     </dd>
+#   </dl>
+# </details>
+#
+###
+
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":heavy_check_mark: __ITEM__"
+comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__"
+comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
+comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
+
+function print_br_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "${format//__ITEM__/${item}}<br/>"
 }
 
-function failed() {
-    echo "    :heavy_multiplication_x: ${1}"
+function print_br_item2() {
+    format="${1}"
+    item="${2}"
+    item2="${3}"
+    format1="${format//__ITEM__/${item}}"
+    echo -n "${format1//__ITEM2__/${item2}}<br/>"
 }
 
-if [[ ${SLURM} -eq 1 ]]; then
-    succeeded "job output file <code>${job_out}</code>" >> ${job_result_file}
-else
-    failed "no job output file matching <code>${GP_slurm_out}</code>" >> ${job_result_file}
-fi
+function print_code_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<code>${format//__ITEM__/${item}}</code>"
+}
 
-if [[ ${ERROR} -eq 0 ]]; then
-    succeeded "no message matching <code>${GP_error}</code>" >> ${job_result_file}
-else
-    failed "found message matching <code>${GP_error}</code>" >> ${job_result_file}
-fi
+function print_dd_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<dd>${format//__ITEM__/${item}}</dd>"
+}
 
-if [[ ${FAILED} -eq 0 ]]; then
-    succeeded "no message matching <code>${GP_failed}</code>" >> ${job_result_file}
-else
-    failed "found message matching <code>${GP_failed}</code>" >> ${job_result_file}
-fi
+function print_list_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<li>${format//__ITEM__/${item}}</li>"
+}
 
-if [[ ${MISSING} -eq 0 ]]; then
-    succeeded "no message matching <code>${GP_req_missing}</code>" >> ${job_result_file}
-else
-    failed "found message matching <code>${GP_req_missing}</code>" >> ${job_result_file}
-fi
+function print_pre_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "<pre>${format//__ITEM__/${item}}</pre>"
+}
 
-if [[ ${NO_MISSING} -eq 1 ]]; then
-    succeeded "found message(s) matching <code>${GP_no_missing}</code>" >> ${job_result_file}
-else
-    failed "no message matching <code>${GP_no_missing}</code>" >> ${job_result_file}
-fi
+function success() {
+    format="${comment_success_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
 
-if [[ ${TGZ} -eq 1 ]]; then
-    succeeded "found message matching <code>${GP_tgz_created}</code>" >> ${job_result_file}
-else
-    failed "no message matching <code>${GP_tgz_created}</code>" >> ${job_result_file}
-fi
+function failure() {
+    format="${comment_failure_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
+function add_detail() {
+    actual=${1}
+    expected=${2}
+    success_msg="${3}"
+    failure_msg="${4}"
+    if [[ ${actual} -eq ${expected} ]]; then
+        success "${success_msg}"
+    else
+        failure "${failure_msg}"
+    fi
+}
+
+echo "[RESULT]" > ${job_result_file}
+echo -n "comment_description = " >> ${job_result_file}
+
+# construct values for placeholders in comment_template:
+# - __SUMMARY_FMT__ -> variable $comment_summary
+# - __DETAILS_FMT__ -> variable $comment_details
+# - __ARTEFACTS_FMT__ -> variable $comment_artefacts
 
-echo "artefacts =" >> ${job_result_file}
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 
+# first construct comment_details_list, abbreviated CoDeList
+# then use it to set comment_details
+CoDeList=""
+
+success_msg="job output file <code>${job_out}</code>"
+failure_msg="no job output file matching <code>${GP_slurm_out}</code>"
+CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_error}</code>"
+failure_msg="found message matching <code>${GP_error}</code>"
+CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_failed}</code>"
+failure_msg="found message matching <code>${GP_failed}</code>"
+CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_req_missing}</code>"
+failure_msg="found message matching <code>${GP_req_missing}</code>"
+CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="found message(s) matching <code>${GP_no_missing}</code>"
+failure_msg="no message matching <code>${GP_no_missing}</code>"
+CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="found message matching <code>${GP_tgz_created}</code>"
+failure_msg="no message matching <code>${GP_tgz_created}</code>"
+CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+
+
+# first construct comment_artefacts_list, abbreviated CoArList
+# then use it to set comment_artefacts
+CoArList=""
+
+# TARBALL should only contain a single tarball
 if [[ ! -z ${TARBALL} ]]; then
-    echo "    ${TARBALL}" >> ${job_result_file}
+    # TODO add tarball details: size, num entries, modules, software pkgs, misc
+    # <dd>
+    #   <details>
+    #     <summary><code>eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz</code></summary>
+    #     size: 234 MiB (245366784 bytes)<br/>
+    #     entries: 1234<br/>
+    #     modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_<br/>
+    #     <pre>
+    #       GCC/9.3.0.lua<br/>
+    #       GCC/10.3.0.lua<br/>
+    #       OpenSSL/1.1.lua
+    #     </pre>
+    #     software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_
+    #     <pre>
+    #       GCC/9.3.0/<br/>
+    #       CMake/3.20.1-GCCcore-10.3.0/<br/>
+    #       OpenMPI/4.1.1-GCC-10.3.0/
+    #     </pre>
+    #     other under _2023.04/software/linux/x86_64/intel/cascadelake/_
+    #     <pre>
+    #       .lmod/cache/spiderT.lua<br/>
+    #       .lmod/cache/spiderT.luac_5.1<br/>
+    #       .lmod/cache/timestamp
+    #     </pre>
+    #   </details>
+    # </dd>
+    size="$(stat --dereference --printf=%s ${TARBALL})"
+    size_mib=$((${size} >> 20))
+    tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
+    tar tf ${TARBALL} > ${tmpfile}
+    entries=$(wc -l ${tmpfile})
+    # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
+    # 2023.04/software/linux/x86_64/intel/skylake_avx512
+    # repo_version = 2022.11
+    # software
+    # os_type = linux
+    # software_subdir = aarch64/generic
+    repo_version=$(cfg_get_value "repository" "repo_version")
+    os_type=$(cfg_get_value "architecture" "os_type")
+    software_subdir=$(cfg_get_value "architecture" "software_subdir")
+    prefix="${repo_version}/software/${os_type}/${software_subdir}"
+    modules_entries=$(grep "${prefix}/modules" ${tmpfile})
+    software_entries=$(grep "${prefix}/software" ${tmpfile})
+    lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile})
+    other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
+    modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@')
+    software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{print $1 "/" $2}' | sort -u)
+    lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@")
+
+    artefact_summary="<summary>$(print_code_item '__ITEM__' ${TARBALL})</summary>"
+    CoArList=""
+    CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
+    CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})"
+    CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
+    CoArList="${CoArList}<pre>"
+    if [[ ! -z ${modules} ]]; then
+        for mod in $(echo "${modules}" | sort); do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
+        done
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
+    fi
+    CoArList="${CoArList}</pre>"
+    CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
+    CoArList="${CoArList}<pre>"
+    if [[ ! -z ${software_pkgs} ]]; then
+        for sw_pkg in $(echo "${software_pkgs}" | sort); do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
+        done
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+    fi
+    CoArList="${CoArList}</pre>"
+    CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})"
+    CoArList="${CoArList}<pre>"
+    if [[ ! -z ${other_entries} ]]; then
+        for other in $(echo "${other_entries}" | sort); do
+            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
+        done
+    else
+        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
+    fi
+    CoArList="${CoArList}</pre>"
+else
+    CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')"
 fi
 
+comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}"
+comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}"
+comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_DETAILS_LIST__/${comment_artefacts_details}}"
+
+# now put all pieces together creating comment_details from comment_template
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}}
+
+echo "${comment_description}" >> ${job_result_file}
+
+# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts
+# - this should make use of subsequent steps such as deploying a tarball more
+#   efficient
+echo "status = ${status}" >> ${job_result_file}
+echo "artefacts = " >> ${job_result_file}
+echo "${TARBALL}" | sed -e 's/^/    /g' >> ${job_result_file}
+
 exit 0

From 61f8280af7292c3fcc98c69589775141583fd103 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 18 Jun 2023 04:56:18 +0200
Subject: [PATCH 11/11] update and bug fixes

---
 bot/check-result.sh | 41 +++++++++++++++++++++--------------------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/bot/check-result.sh b/bot/check-result.sh
index 78c850d3b4..04a23962e2 100755
--- a/bot/check-result.sh
+++ b/bot/check-result.sh
@@ -105,7 +105,7 @@ job_out=$(ls ${job_dir} | grep "${GP_slurm_out}")
 ERROR=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_error='ERROR: '
-  grep_out=$(grep "${GP_error}" ${job_dir}/${job_out})
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
@@ -115,7 +115,7 @@ fi
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_failed='FAILED: '
-  grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out})
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
@@ -125,7 +125,7 @@ fi
 MISSING=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_req_missing=' required modules missing:'
-  grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out})
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}")
   [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
@@ -135,7 +135,7 @@ fi
 NO_MISSING=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_no_missing='No missing installations'
-  grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out})
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}")
   [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'"
@@ -145,8 +145,8 @@ fi
 TGZ=-1
 TARBALL=
 if [[ ${SLURM} -eq 1 ]]; then
-  GP_tgz_created="tar.gz created!"
-  grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out})
+  GP_tgz_created="\.tar\.gz created!"
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
   if [[ $? -eq 0 ]]; then
       TGZ=1
       TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@')
@@ -404,7 +404,7 @@ if [[ ! -z ${TARBALL} ]]; then
     size_mib=$((${size} >> 20))
     tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
     tar tf ${TARBALL} > ${tmpfile}
-    entries=$(wc -l ${tmpfile})
+    entries=$(cat ${tmpfile} | wc -l)
     # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
     # 2023.04/software/linux/x86_64/intel/skylake_avx512
     # repo_version = 2022.11
@@ -419,8 +419,9 @@ if [[ ! -z ${TARBALL} ]]; then
     software_entries=$(grep "${prefix}/software" ${tmpfile})
     lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile})
     other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
-    modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@')
-    software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{print $1 "/" $2}' | sort -u)
+    other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
+    modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
+    software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
     lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@")
 
     artefact_summary="<summary>$(print_code_item '__ITEM__' ${TARBALL})</summary>"
@@ -430,9 +431,9 @@ if [[ ! -z ${TARBALL} ]]; then
     CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
     CoArList="${CoArList}<pre>"
     if [[ ! -z ${modules} ]]; then
-        for mod in $(echo "${modules}" | sort); do
-            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
-        done
+        while IFS= read -r mod ; do
+            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${mod})"
+        done <<< "${modules}"
     else
         CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
     fi
@@ -440,19 +441,19 @@ if [[ ! -z ${TARBALL} ]]; then
     CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
     CoArList="${CoArList}<pre>"
     if [[ ! -z ${software_pkgs} ]]; then
-        for sw_pkg in $(echo "${software_pkgs}" | sort); do
-            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
-        done
+        while IFS= read -r sw_pkg ; do
+            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${sw_pkg})"
+        done <<< "${software_pkgs}"
     else
         CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
     fi
     CoArList="${CoArList}</pre>"
     CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})"
     CoArList="${CoArList}<pre>"
-    if [[ ! -z ${other_entries} ]]; then
-        for other in $(echo "${other_entries}" | sort); do
-            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
-        done
+    if [[ ! -z ${other_shortened} ]]; then
+        while IFS= read -r other ; do
+            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${other})"
+        done <<< "${other_shortened}"
     else
         CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
     fi
@@ -463,7 +464,7 @@ fi
 
 comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}"
 comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}"
-comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_DETAILS_LIST__/${comment_artefacts_details}}"
+comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}"
 
 # now put all pieces together creating comment_details from comment_template
 comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}