diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 04195dd619..d3906a0f9f 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -10,7 +10,7 @@ jobs:
fail-fast: false
matrix:
EESSI_VERSION:
- - 2021.12
+ - 2023.06
EESSI_SOFTWARE_SUBDIR:
- aarch64/generic
- aarch64/graviton2
@@ -20,6 +20,8 @@ jobs:
- x86_64/intel/haswell
- x86_64/intel/skylake_avx512
- x86_64/generic
+ EASYSTACK_FILE:
+ - eessi-2023.06-eb-4.7.2-2021a.yml
steps:
- name: Check out software-layer repository
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
@@ -31,17 +33,17 @@ jobs:
cvmfs_http_proxy: DIRECT
cvmfs_repositories: pilot.eessi-hpc.org
- - name: Test check_missing_installations.sh script
- run: |
- source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
- module load EasyBuild
- eb --version
- export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
- export EESSI_OS_TYPE=linux
- export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
- env | grep ^EESSI | sort
- echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
- ./check_missing_installations.sh
+# - name: Test check_missing_installations.sh script
+# run: |
+# source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+# module load EasyBuild
+# eb --version
+# export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+# export EESSI_OS_TYPE=linux
+# export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+# env | grep ^EESSI | sort
+# echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
+# ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}
- name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
run: |
@@ -52,18 +54,15 @@ jobs:
export EESSI_OS_TYPE=linux
export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
env | grep ^EESSI | sort
- echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
- echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
- echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
- echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
- echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
- tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+ echo "modify easystack file by adding a missing package (GCC/8.3.0)"
+ echo " - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}}
+ tail -n 5 ${{matrix.EASYSTACK_FILE}}
# note, check_missing_installations.sh exits 1 if a package was
# missing, which is intepreted as false (exit code based, not
# boolean logic), hence when the script exits 0 if no package was
# missing it is interpreted as true, thus the test did not capture
# the missing package
- if ./check_missing_installations.sh; then
+ if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then
echo "did NOT capture missing package; test FAILED"
exit 1
else
diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml
index 5c0b3893ae..8ad4928993 100644
--- a/.github/workflows/tests_scripts.yml
+++ b/.github/workflows/tests_scripts.yml
@@ -92,6 +92,6 @@ jobs:
# since create_directory_tarballs.sh must be accessible from within build container
cp -a * /tmp/
cd /tmp
- ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12
+ ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06
# check if tarballs have been produced
ls -l *.tar.gz
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index d9bcf20231..8427ebe374 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -50,28 +50,28 @@ set -- "${POSITIONAL_ARGS[@]}"
TOPDIR=$(dirname $(realpath $0))
-source $TOPDIR/scripts/utils.sh
+source ${TOPDIR}/scripts/utils.sh
-# honor $TMPDIR if it is already defined, use /tmp otherwise
-if [ -z $TMPDIR ]; then
- export WORKDIR=/tmp/$USER
+# honor ${TMPDIR} if it is already defined, use /tmp otherwise
+if [ -z ${TMPDIR} ]; then
+ export WORKDIR=/tmp/${USER}
else
- export WORKDIR=$TMPDIR/$USER
+ export WORKDIR=${TMPDIR}/${USER}
fi
TMPDIR=$(mktemp -d)
echo ">> Setting up environment..."
-source $TOPDIR/init/minimal_eessi_env
+source ${TOPDIR}/init/minimal_eessi_env
-if [ -d $EESSI_CVMFS_REPO ]; then
- echo_green "$EESSI_CVMFS_REPO available, OK!"
+if [ -d ${EESSI_CVMFS_REPO} ]; then
+ echo_green "${EESSI_CVMFS_REPO} available, OK!"
else
- fatal_error "$EESSI_CVMFS_REPO is not available!"
+ fatal_error "${EESSI_CVMFS_REPO} is not available!"
fi
-# make sure we're in Prefix environment by checking $SHELL
+# make sure we're in Prefix environment by checking ${SHELL}
if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
else
@@ -79,30 +79,30 @@ else
fi
# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+export PYTHONPYCACHEPREFIX=${TMPDIR}/pycache
DETECTION_PARAMETERS=''
GENERIC=0
EB='eb'
-if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
+if [[ "${EASYBUILD_OPTARCH}" == "GENERIC" ]]; then
echo_yellow ">> GENERIC build requested, taking appropriate measures!"
- DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
+ DETECTION_PARAMETERS="${DETECTION_PARAMETERS} --generic"
GENERIC=1
EB='eb --optarch=GENERIC'
fi
echo ">> Determining software subdirectory to use for current build host..."
-if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
- export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
- echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+if [ -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+ export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 ${TOPDIR}/eessi_software_subdir.py ${DETECTION_PARAMETERS})
+ echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py ${DETECTION_PARAMETERS}' script"
else
echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
fi
-# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+# Set all the EESSI environment variables (respecting ${EESSI_SOFTWARE_SUBDIR_OVERRIDE})
# $EESSI_SILENT - don't print any messages
# $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source ${TOPDIR}/init/eessi_environment_variables
if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
fatal_error "Failed to determine software subdirectory?!"
@@ -113,9 +113,9 @@ else
fi
echo ">> Initializing Lmod..."
-source $EPREFIX/usr/share/Lmod/init/bash
-ml_version_out=$TMPDIR/ml.out
-ml --version &> $ml_version_out
+source ${EPREFIX}/usr/share/Lmod/init/bash
+ml_version_out=${TMPDIR}/ml.out
+ml --version &> ${ml_version_out}
if [[ $? -eq 0 ]]; then
echo_green ">> Found Lmod ${LMOD_VERSION}"
else
@@ -123,262 +123,50 @@ else
fi
echo ">> Configuring EasyBuild..."
-source $TOPDIR/configure_easybuild
+source ${TOPDIR}/configure_easybuild
echo ">> Setting up \$MODULEPATH..."
# make sure no modules are loaded
module --force purge
# ignore current $MODULEPATH entirely
-module unuse $MODULEPATH
-module use $EASYBUILD_INSTALLPATH/modules/all
+module unuse ${MODULEPATH}
+module use ${EASYBUILD_INSTALLPATH}/modules/all
if [[ -z ${MODULEPATH} ]]; then
fatal_error "Failed to set up \$MODULEPATH?!"
else
echo_green ">> MODULEPATH set up: ${MODULEPATH}"
fi
-REQ_EB_VERSION='4.5.0'
+for eb_version in '4.7.2'; do
+ # load EasyBuild module (will be installed if it's not available yet)
+ source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
-# load EasyBuild module (will be installed if it's not available yet)
-source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION}
+ echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
-echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."
+ for gen in '2021a'; do
-# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up,
-# see https://github.com/EESSI/software-layer/issues/123
-# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557
-ok_msg="Java installed, off to a good (?) start!"
-fail_msg="Failed to install Java, woopsie..."
-$EB Java-11.eb --robot --include-easyblocks-from-pr 2557
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+ es="eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-${gen}.yml"
-# install GCC for foss/2020a
-export GCC_EC="GCC-9.3.0.eb"
-echo ">> Starting slow with ${GCC_EC}..."
-ok_msg="${GCC_EC} installed, yippy! Off to a good start..."
-fail_msg="Installation of ${GCC_EC} failed!"
-# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453,
-# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place
-$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+ if [ -f ${es} ]; then
+ echo_green "Feeding easystack file ${es} to EasyBuild..."
-# install CMake with custom easyblock that patches CMake when --sysroot is used
-echo ">> Install CMake with fixed easyblock to take into account --sysroot"
-ok_msg="CMake installed!"
-fail_msg="Installation of CMake failed, what the ..."
-$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+ ${EB} --easystack ${TOPDIR}/${es} --missing && ${EB} --easystack ${TOPDIR}/${es} --robot
-# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946
-echo ">> Installing OpenBLAS..."
-ok_msg="Done with OpenBLAS!"
-fail_msg="Installation of OpenBLAS failed!"
-if [[ $GENERIC -eq 1 ]]; then
- echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS."
- openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946"
-else
- openblas_include_easyblocks_from_pr=''
-fi
-$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OpenMPI..."
-ok_msg="OpenMPI installed, w00!"
-fail_msg="Installation of OpenMPI failed, that's not good..."
-$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# install Python
-echo ">> Install Python 2.7.18 and Python 3.8.2..."
-ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!"
-fail_msg="Installation of Python failed, oh no..."
-$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Perl..."
-ok_msg="Perl installed, making progress..."
-fail_msg="Installation of Perl failed, this never happens..."
-# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
-# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
-$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Qt5..."
-ok_msg="Qt5 installed, phieuw, that was a big one!"
-fail_msg="Installation of Qt5 failed, that's frustrating..."
-$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# skip test step when installing SciPy-bundle on aarch64,
-# to dance around problem with broken numpy tests;
-# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959
-echo ">> Installing SciPy-bundle"
-ok_msg="SciPy-bundle installed, yihaa!"
-fail_msg="SciPy-bundle installation failed, bummer..."
-SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb
-if [[ "$(uname -m)" == "aarch64" ]]; then
- $EB $SCIPY_EC --robot --skip-test-step
-else
- $EB $SCIPY_EC --robot
-fi
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing GROMACS..."
-ok_msg="GROMACS installed, wow!"
-fail_msg="Installation of GROMACS failed, damned..."
-$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)!
-# 32GB is sufficient to build with 16 cores
-echo ">> Installing OpenFOAM (twice!)..."
-ok_msg="OpenFOAM installed, now we're talking!"
-fail_msg="Installation of OpenFOAM failed, we were so close..."
-$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then
- echo ">> Installing QuantumESPRESSO..."
- ok_msg="QuantumESPRESSO installed, let's go quantum!"
- fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!"
- $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot
- check_exit_code $? "${ok_msg}" "${fail_msg}"
-fi
-
-echo ">> Installing R 4.0.0 (better be patient)..."
-ok_msg="R installed, wow!"
-fail_msg="Installation of R failed, so sad..."
-$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Bioconductor 3.11 bundle..."
-ok_msg="Bioconductor installed, enjoy!"
-fail_msg="Installation of Bioconductor failed, that's annoying..."
-$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing TensorFlow 2.3.1..."
-ok_msg="TensorFlow 2.3.1 installed, w00!"
-fail_msg="Installation of TensorFlow failed, why am I not surprised..."
-$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Horovod 0.21.3..."
-ok_msg="Horovod installed! Go do some parallel training!"
-fail_msg="Horovod installation failed. There comes the headache..."
-$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then
-
- echo ">> Installing code-server 3.7.3..."
- ok_msg="code-server 3.7.3 installed, now you can use VS Code!"
- fail_msg="Installation of code-server failed, that's going to be hard to fix..."
- $EB code-server-3.7.3.eb --robot
- check_exit_code $? "${ok_msg}" "${fail_msg}"
-fi
-
-echo ">> Installing RStudio-Server 1.3.1093..."
-ok_msg="RStudio-Server installed, enjoy!"
-fail_msg="Installation of RStudio-Server failed, might be OS deps..."
-$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OSU-Micro-Benchmarks 5.6.3..."
-ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
-fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
-$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Spark 3.1.1..."
-ok_msg="Spark installed, set off the fireworks!"
-fail_msg="Installation of Spark failed, no fireworks this time..."
-$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing IPython 7.15.0..."
-ok_msg="IPython installed, launch your Jupyter Notebooks!"
-fail_msg="Installation of IPython failed, that's unexpected..."
-$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing WRF 3.9.1.1..."
-ok_msg="WRF installed, it's getting hot in here!"
-fail_msg="Installation of WRF failed, that's unexpected..."
-OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing R 4.1.0 (better be patient)..."
-ok_msg="R installed, wow!"
-fail_msg="Installation of R failed, so sad..."
-$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Nextflow 22.10.1..."
-ok_msg="Nextflow installed, the work must flow..."
-fail_msg="Installation of Nextflow failed, that's unexpected..."
-$EB -r --from-pr 16531 Nextflow-22.10.1.eb
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..."
-ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
-fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
-$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing EasyBuild 4.5.1..."
-ok_msg="EasyBuild v4.5.1 installed"
-fail_msg="EasyBuild v4.5.1 failed to install"
-$EB --from-pr 14545 --include-easyblocks-from-pr 2805
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1
-check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1"
-
-echo ">> Installing SciPy-bundle with foss/2021a..."
-ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age"
-fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..."
-# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454
-# which includes patch to fix installation with recent Linux kernel headers
-$EB --from-pr 14454 GCCcore-10.3.0.eb --robot
-# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
-# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
-$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640
-# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set
-# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248
-$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
-# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584
-# that includes patch to fix bootstrap problem when using alternate sysroot
-$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
-# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
-# which includes a patch to fix installation on POWER
-$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
-# ignore failing FlexiBLAS tests when building on POWER;
-# some tests are failing due to a segmentation fault due to "invalid memory reference",
-# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476;
-# using -fstack-protector-strong -fstack-clash-protection should fix that,
-# but it doesn't for some reason when building for ppc64le/generic...
-if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then
- $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure
-else
- $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb
-fi
-
-$EB SciPy-bundle-2021.05-foss-2021a.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-### add packages here
+ $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es}
+ else
+ fatal_error "Easystack file ${es} not found!"
+ fi
+ done
+done
echo ">> Creating/updating Lmod cache..."
export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"
-if [ ! -f $LMOD_RC ]; then
- python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
- check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC"
+if [ ! -f ${LMOD_RC} ]; then
+ python3 ${TOPDIR}/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
+ check_exit_code $? "${LMOD_RC} created" "Failed to create ${LMOD_RC}"
fi
-$TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
-
-$TOPDIR/check_missing_installations.sh
+${TOPDIR}/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
echo ">> Cleaning up ${TMPDIR}..."
rm -r ${TMPDIR}
diff --git a/bot/check-result.sh b/bot/check-result.sh
new file mode 100755
index 0000000000..04a23962e2
--- /dev/null
+++ b/bot/check-result.sh
@@ -0,0 +1,483 @@
+#!/bin/bash
+#
+# Script to check the result of building the EESSI software layer.
+# Intended use is that it is called by a (batch) job running on a compute
+# node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# result cases
+
+# - SUCCESS (all of)
+# - working directory contains slurm-JOBID.out file
+# - working directory contains eessi*tar.gz
+# - no message ERROR
+# - no message FAILED
+# - no message ' required modules missing:'
+# - one or more of 'No missing modules!'
+# - message regarding created tarball
+# - FAILED (one of ... implemented as NOT SUCCESS)
+# - no slurm-JOBID.out file
+# - no tarball
+# - message with ERROR
+# - message with FAILED
+# - message with ' required modules missing:'
+# - no message regarding created tarball
+
+# stop as soon as something fails
+# set -e
+
+TOPDIR=$(dirname $(realpath $0))
+
+source ${TOPDIR}/../scripts/utils.sh
+source ${TOPDIR}/../scripts/cfg_files.sh
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+ echo_red "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+else
+ echo "bot/check-result.sh: showing ${JOB_CFG_FILE} from software-layer side"
+ cat ${JOB_CFG_FILE}
+
+ echo "bot/check-result.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+ cfg_load ${JOB_CFG_FILE}
+fi
+
+display_help() {
+ echo "usage: $0 [OPTIONS]"
+ echo " OPTIONS:"
+ echo " -h | --help - display this usage information [default: false]"
+ echo " -v | --verbose - display more information [default: false]"
+}
+
+# set defaults for command line arguments
+VERBOSE=0
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+ case $1 in
+ -h|--help)
+ display_help
+ exit 0
+ ;;
+ -v|--verbose)
+ VERBOSE=1
+ shift 1
+ ;;
+ --)
+ shift
+ POSITIONAL_ARGS+=("$@") # save positional args
+ break
+ ;;
+ -*|--*)
+ fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
+ ;;
+ *) # No more options
+ POSITIONAL_ARGS+=("$1") # save positional arg
+ shift
+ ;;
+ esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+job_dir=${PWD}
+
+[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
+
+GP_slurm_out="slurm-${SLURM_JOB_ID}.out"
+job_out=$(ls ${job_dir} | grep "${GP_slurm_out}")
+[[ $? -eq 0 ]] && SLURM=1 || SLURM=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'"
+
+ERROR=-1
+if [[ ${SLURM} -eq 1 ]]; then
+ GP_error='ERROR: '
+ grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
+ [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+ # have to be careful to not add searched for pattern into slurm out file
+ [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+ [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+FAILED=-1
+if [[ ${SLURM} -eq 1 ]]; then
+ GP_failed='FAILED: '
+ grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+ [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+ # have to be careful to not add searched for pattern into slurm out file
+ [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+ [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+MISSING=-1
+if [[ ${SLURM} -eq 1 ]]; then
+ GP_req_missing=' required modules missing:'
+ grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}")
+ [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
+ # have to be careful to not add searched for pattern into slurm out file
+ [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
+ [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+NO_MISSING=-1
+if [[ ${SLURM} -eq 1 ]]; then
+ GP_no_missing='No missing installations'
+ grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}")
+ [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
+ # have to be careful to not add searched for pattern into slurm out file
+ [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'"
+ [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+TGZ=-1
+TARBALL=
+if [[ ${SLURM} -eq 1 ]]; then
+ GP_tgz_created="\.tar\.gz created!"
+ grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
+ if [[ $? -eq 0 ]]; then
+ TGZ=1
+ TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@')
+ else
+ TGZ=0
+ fi
+ # have to be careful to not add searched for pattern into slurm out file
+ [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
+ [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}"
+[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)"
+[[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+
+job_result_file=_bot_job${SLURM_JOB_ID}.result
+
+if [[ ${SLURM} -eq 1 ]] && \
+ [[ ${ERROR} -eq 0 ]] && \
+ [[ ${FAILED} -eq 0 ]] && \
+ [[ ${MISSING} -eq 0 ]] && \
+ [[ ${NO_MISSING} -eq 1 ]] && \
+ [[ ${TGZ} -eq 1 ]] && \
+ [[ ! -z ${TARBALL} ]]; then
+ # SUCCESS
+ status="SUCCESS"
+ summary=":grin: SUCCESS"
+else
+ # FAILURE
+ status="FAILURE"
+ summary=":cry: FAILURE"
+fi
+
+### Example details/descriptions
+# Note, final string must not contain any line breaks. Below examples include
+# line breaks for the sake of readability.
+#
+# :cry: FAILURE _(click triangle for detailed information)_
+# Details:
+# :heavy_check_mark: job output file slurm-470503.out
+# :heavy_multiplication_x: found message matching ERROR:
+# :heavy_multiplication_x: found message matching FAILED:
+# :heavy_multiplication_x: found message matching required modules missing:
+# :heavy_check_mark: found message(s) matching No missing installations
+# :heavy_check_mark: found message matching tar.gz created!
+# Artefacts:
+# eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
+#
+#
+#
+# :grin: SUCCESS _(click triangle for detailed information)_
+# Details:
+# :heavy_check_mark: job output file slurm-470503.out
+# :heavy_check_mark: found message matching ERROR:
+# :heavy_check_mark: found message matching FAILED:
+# :heavy_check_mark: found message matching required modules missing:
+# :heavy_check_mark: found message(s) matching No missing installations
+# :heavy_check_mark: found message matching tar.gz created!
+# Artefacts:
+# eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz
+#
+#
+#
+# :grin: SUCCESS _(click triangle for detailed information)_
+#
+# - _Details_
+# -
+# :heavy_check_mark: job output file
slurm-4682.out
+# :heavy_check_mark: no message matching ERROR:
+# :heavy_check_mark: no message matching FAILED:
+# :heavy_check_mark: no message matching required modules missing:
+# :heavy_check_mark: found message(s) matching No missing installations
+# :heavy_check_mark: found message matching tar.gz created!
+#
+# - _Artefacts_
+# -
+#
+# eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz
+# size: 234 MiB (245366784 bytes)
+# entries: 1234
+# modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
+#
+# GCC/9.3.0.lua
+# GCC/10.3.0.lua
+# OpenSSL/1.1.lua
+#
+# software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_
+#
+# GCC/9.3.0/
+# CMake/3.20.1-GCCcore-10.3.0/
+# OpenMPI/4.1.1-GCC-10.3.0/
+#
+# other under _2023.04/software/linux/x86_64/intel/cascadelake/_
+#
+# .lmod/cache/spiderT.lua
+# .lmod/cache/spiderT.luac_5.1
+# .lmod/cache/timestamp
+#
+#
+#
+#
+#
+#
+###
+
+# construct and write complete PR comment details: implements third alternative
+comment_template="__SUMMARY_FMT____DETAILS_FMT____ARTEFACTS_FMT__
"
+comment_summary_fmt="__SUMMARY__ _(click triangle for details)_"
+comment_details_fmt="_Details___DETAILS_LIST__"
+comment_success_item_fmt=":heavy_check_mark: __ITEM__"
+comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__"
+comment_artefacts_fmt="_Artefacts___ARTEFACTS_LIST__"
+comment_artefact_details_fmt="__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__ "
+
+function print_br_item() {
+ format="${1}"
+ item="${2}"
+ echo -n "${format//__ITEM__/${item}}
"
+}
+
+function print_br_item2() {
+ format="${1}"
+ item="${2}"
+ item2="${3}"
+ format1="${format//__ITEM__/${item}}"
+ echo -n "${format1//__ITEM2__/${item2}}
"
+}
+
+function print_code_item() {
+ format="${1}"
+ item="${2}"
+ echo -n "${format//__ITEM__/${item}}"
+}
+
+function print_dd_item() {
+ format="${1}"
+ item="${2}"
+ echo -n "${format//__ITEM__/${item}}"
+}
+
+function print_list_item() {
+ format="${1}"
+ item="${2}"
+ echo -n "${format//__ITEM__/${item}}"
+}
+
+function print_pre_item() {
+ format="${1}"
+ item="${2}"
+ echo -n "${format//__ITEM__/${item}}"
+}
+
+function success() {
+ format="${comment_success_item_fmt}"
+ item="$1"
+ print_br_item "${format}" "${item}"
+}
+
+function failure() {
+ format="${comment_failure_item_fmt}"
+ item="$1"
+ print_br_item "${format}" "${item}"
+}
+
+function add_detail() {
+ actual=${1}
+ expected=${2}
+ success_msg="${3}"
+ failure_msg="${4}"
+ if [[ ${actual} -eq ${expected} ]]; then
+ success "${success_msg}"
+ else
+ failure "${failure_msg}"
+ fi
+}
+
+echo "[RESULT]" > ${job_result_file}
+echo -n "comment_description = " >> ${job_result_file}
+
+# construct values for placeholders in comment_template:
+# - __SUMMARY_FMT__ -> variable $comment_summary
+# - __DETAILS_FMT__ -> variable $comment_details
+# - __ARTEFACTS_FMT__ -> variable $comment_artefacts
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# first construct comment_details_list, abbreviated CoDeList
+# then use it to set comment_details
+CoDeList=""
+
+success_msg="job output file ${job_out}"
+failure_msg="no job output file matching ${GP_slurm_out}"
+CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching ${GP_error}"
+failure_msg="found message matching ${GP_error}"
+CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching ${GP_failed}"
+failure_msg="found message matching ${GP_failed}"
+CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching ${GP_req_missing}"
+failure_msg="found message matching ${GP_req_missing}"
+CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="found message(s) matching ${GP_no_missing}"
+failure_msg="no message matching ${GP_no_missing}"
+CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="found message matching ${GP_tgz_created}"
+failure_msg="no message matching ${GP_tgz_created}"
+CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+
+
+# first construct comment_artefacts_list, abbreviated CoArList
+# then use it to set comment_artefacts
+CoArList=""
+
+# TARBALL should only contain a single tarball
+if [[ ! -z ${TARBALL} ]]; then
+ # TODO add tarball details: size, num entries, modules, software pkgs, misc
+ #
+ #
+ # eessi-2023.04-software-linux-x86_64-generic-1682696567.tar.gz
+ # size: 234 MiB (245366784 bytes)
+ # entries: 1234
+ # modules under _2023.04/software/linux/x86_64/intel/cascadelake/modules/all/_
+ #
+ # GCC/9.3.0.lua
+ # GCC/10.3.0.lua
+ # OpenSSL/1.1.lua
+ #
+ # software under _2023.04/software/linux/x86_64/intel/cascadelake/software/_
+ #
+ # GCC/9.3.0/
+ # CMake/3.20.1-GCCcore-10.3.0/
+ # OpenMPI/4.1.1-GCC-10.3.0/
+ #
+ # other under _2023.04/software/linux/x86_64/intel/cascadelake/_
+ #
+ # .lmod/cache/spiderT.lua
+ # .lmod/cache/spiderT.luac_5.1
+ # .lmod/cache/timestamp
+ #
+ #
+ #
+ size="$(stat --dereference --printf=%s ${TARBALL})"
+ size_mib=$((${size} >> 20))
+ tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
+ tar tf ${TARBALL} > ${tmpfile}
+ entries=$(cat ${tmpfile} | wc -l)
+ # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
+ # 2023.04/software/linux/x86_64/intel/skylake_avx512
+ # repo_version = 2022.11
+ # software
+ # os_type = linux
+ # software_subdir = aarch64/generic
+ repo_version=$(cfg_get_value "repository" "repo_version")
+ os_type=$(cfg_get_value "architecture" "os_type")
+ software_subdir=$(cfg_get_value "architecture" "software_subdir")
+ prefix="${repo_version}/software/${os_type}/${software_subdir}"
+ modules_entries=$(grep "${prefix}/modules" ${tmpfile})
+ software_entries=$(grep "${prefix}/software" ${tmpfile})
+ lmod_entries=$(grep "${prefix}/.lmod/cache" ${tmpfile})
+ other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
+ other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
+ modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
+ software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
+ lmod_shortened=$(echo "${lmod_entries}" | sed -e "s@${prefix}/@@")
+
+ artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})"
+ CoArList=""
+ CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
+ CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})"
+ CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
+ CoArList="${CoArList}"
+ if [[ ! -z ${modules} ]]; then
+ while IFS= read -r mod ; do
+ CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
+ done <<< "${modules}"
+ else
+ CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
+ fi
+ CoArList="${CoArList}"
+ CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
+ CoArList="${CoArList}"
+ if [[ ! -z ${software_pkgs} ]]; then
+ while IFS= read -r sw_pkg ; do
+ CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
+ done <<< "${software_pkgs}"
+ else
+ CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+ fi
+ CoArList="${CoArList}"
+ CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})"
+ CoArList="${CoArList}"
+ if [[ ! -z ${other_shortened} ]]; then
+ while IFS= read -r other ; do
+ CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
+ done <<< "${other_shortened}"
+ else
+ CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
+ fi
+ CoArList="${CoArList}"
+else
+ CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')"
+fi
+
+comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}"
+comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}"
+comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}"
+
+# now put all pieces together creating comment_details from comment_template
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}}
+
+echo "${comment_description}" >> ${job_result_file}
+
+# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts
+# - this should make use of subsequent steps such as deploying a tarball more
+# efficient
+echo "status = ${status}" >> ${job_result_file}
+echo "artefacts = " >> ${job_result_file}
+echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file}
+
+exit 0
diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 4a5316c09f..3627d1d0b5 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Script to check for missing installations in EESSI pilot software stack (version 2021.12)
+# Script to check for missing installations in EESSI pilot software stack (version 2023.06)
#
# author: Kenneth Hoste (@boegel)
# author: Thomas Roeblitz (@trz42)
@@ -10,10 +10,11 @@
TOPDIR=$(dirname $(realpath $0))
-if [ -z ${EESSI_PILOT_VERSION} ]; then
- echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2
+if [ $# -ne 1 ]; then
+ echo "ERROR: Usage: $0 " >&2
exit 1
fi
+easystack=$1
LOCAL_TMPDIR=$(mktemp -d)
@@ -23,9 +24,7 @@ source $TOPDIR/configure_easybuild
echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
-# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
-# PR #16531: Nextflow-22.10.1.eb
-${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
+${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out}
exit_code=${PIPESTATUS[0]}
ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
diff --git a/configure_easybuild b/configure_easybuild
index 245553f342..8e79f3a35a 100644
--- a/configure_easybuild
+++ b/configure_easybuild
@@ -13,7 +13,6 @@ export EASYBUILD_ZIP_LOGS=bzip2
export EASYBUILD_RPATH=1
export EASYBUILD_FILTER_ENV_VARS=LD_LIBRARY_PATH
-export EASYBUILD_READ_ONLY_INSTALLDIR=1
# assume that eb_hooks.py is located in same directory as this script (configure_easybuild)
TOPDIR=$(dirname $(realpath $BASH_SOURCE))
@@ -36,3 +35,6 @@ fi
export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER
export EASYBUILD_MODULE_EXTENSIONS=1
+
+# need to enable use of experimental features, since we're using easystack files
+export EASYBUILD_EXPERIMENTAL=1
diff --git a/eb_hooks.py b/eb_hooks.py
index 2fba925b01..8df5e076e0 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -11,6 +11,14 @@
from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features
from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
+# prefer importing LooseVersion from easybuild.tools, but fall back to distutils
+# in case EasyBuild <= 4.7.0 is used
+try:
+ from easybuild.tools import LooseVersion
+except ImportError:
+ from distutils.version import LooseVersion
+
+
EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
@@ -46,23 +54,6 @@ def get_rpath_override_dirs(software_name):
return rpath_injection_dirs
-def parse_hook(ec, *args, **kwargs):
- """Main parse hook: trigger custom functions based on software name."""
-
- # determine path to Prefix installation in compat layer via $EPREFIX
- eprefix = get_eessi_envvar('EPREFIX')
-
- if ec.name in PARSE_HOOKS:
- PARSE_HOOKS[ec.name](ec, eprefix)
-
-
-def pre_configure_hook(self, *args, **kwargs):
- """Main pre-configure hook: trigger custom functions based on software name."""
-
- if self.name in PRE_CONFIGURE_HOOKS:
- PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
-
-
def pre_prepare_hook(self, *args, **kwargs):
"""Main pre-prepare hook: trigger custom functions."""
@@ -91,10 +82,24 @@ def pre_prepare_hook(self, *args, **kwargs):
mpi_family, rpath_override_dirs)
-def gcc_postprepare(self, *args, **kwargs):
+def post_prepare_hook(self, *args, **kwargs):
+ """Main post-prepare hook: trigger custom functions."""
+
+ if hasattr(self, EESSI_RPATH_OVERRIDE_ATTR):
+ # Reset the value of 'rpath_override_dirs' now that we are finished with it
+ update_build_option('rpath_override_dirs', getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
+ print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
+ delattr(self, EESSI_RPATH_OVERRIDE_ATTR)
+
+ if self.name in POST_PREPARE_HOOKS:
+ POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
+
+
+def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs):
"""
Post-configure hook for GCCcore:
- - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu'
+ - copy RPATH wrapper script for linker commands to also have a wrapper in
+ place with system type prefix like 'x86_64-pc-linux-gnu'
"""
if self.name == 'GCCcore':
config_guess = obtain_config_guess()
@@ -121,20 +126,18 @@ def gcc_postprepare(self, *args, **kwargs):
else:
raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!")
-def post_prepare_hook(self, *args, **kwargs):
- """Main post-prepare hook: trigger custom functions."""
- if hasattr(self, EESSI_RPATH_OVERRIDE_ATTR):
- # Reset the value of 'rpath_override_dirs' now that we are finished with it
- update_build_option('rpath_override_dirs', getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
- print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR))
- delattr(self, EESSI_RPATH_OVERRIDE_ATTR)
+def parse_hook(ec, *args, **kwargs):
+ """Main parse hook: trigger custom functions based on software name."""
- if self.name in POST_PREPARE_HOOKS:
- POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
+ # determine path to Prefix installation in compat layer via $EPREFIX
+ eprefix = get_eessi_envvar('EPREFIX')
+ if ec.name in PARSE_HOOKS:
+ PARSE_HOOKS[ec.name](ec, eprefix)
-def cgal_toolchainopts_precise(ec, eprefix):
+
+def parse_hook_cgal_toolchainopts_precise(ec, eprefix):
"""Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
if ec.name == 'CGAL':
if get_cpu_architecture() == POWER:
@@ -147,7 +150,7 @@ def cgal_toolchainopts_precise(ec, eprefix):
raise EasyBuildError("CGAL-specific hook triggered for non-CGAL easyconfig?!")
-def fontconfig_add_fonts(ec, eprefix):
+def parse_hook_fontconfig_add_fonts(ec, eprefix):
"""Inject --with-add-fonts configure option for fontconfig."""
if ec.name == 'fontconfig':
# make fontconfig aware of fonts included with compat layer
@@ -158,7 +161,7 @@ def fontconfig_add_fonts(ec, eprefix):
raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!")
-def ucx_eprefix(ec, eprefix):
+def parse_hook_ucx_eprefix(ec, eprefix):
"""Make UCX aware of compatibility layer via additional configuration options."""
if ec.name == 'UCX':
ec.update('configopts', '--with-sysroot=%s' % eprefix)
@@ -174,7 +177,19 @@ def pre_configure_hook(self, *args, **kwargs):
PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
-def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs):
+def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs):
+ """
+ Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC
+ """
+ if self.name == 'OpenBLAS':
+ if build_option('optarch') == OPTARCH_GENERIC:
+ for step in ('build', 'test', 'install'):
+ self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1")
+ else:
+ raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!")
+
+
+def pre_configure_hook_libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs):
"""Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64."""
if self.name == 'libfabric':
if get_cpu_architecture() == X86_64:
@@ -187,7 +202,7 @@ def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs):
raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!")
-def metabat_preconfigure(self, *args, **kwargs):
+def pre_configure_hook_metabat_filtered_zlib_dep(self, *args, **kwargs):
"""
Pre-configure hook for MetaBAT:
- take into account that zlib is a filtered dependency,
@@ -201,7 +216,7 @@ def metabat_preconfigure(self, *args, **kwargs):
raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!")
-def wrf_preconfigure(self, *args, **kwargs):
+def pre_configure_hook_wrf_aarch64(self, *args, **kwargs):
"""
Pre-configure hook for WRF:
- patch arch/configure_new.defaults so building WRF with foss toolchain works on aarch64
@@ -217,17 +232,18 @@ def wrf_preconfigure(self, *args, **kwargs):
PARSE_HOOKS = {
- 'CGAL': cgal_toolchainopts_precise,
- 'fontconfig': fontconfig_add_fonts,
- 'UCX': ucx_eprefix,
+ 'CGAL': parse_hook_cgal_toolchainopts_precise,
+ 'fontconfig': parse_hook_fontconfig_add_fonts,
+ 'UCX': parse_hook_ucx_eprefix,
}
POST_PREPARE_HOOKS = {
- 'GCCcore': gcc_postprepare,
+ 'GCCcore': post_prepare_hook_gcc_prefixed_ld_rpath_wrapper,
}
PRE_CONFIGURE_HOOKS = {
- 'libfabric': libfabric_disable_psm3_x86_64_generic,
- 'MetaBAT': metabat_preconfigure,
- 'WRF': wrf_preconfigure,
+ 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic,
+ 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep,
+ 'OpenBLAS': pre_configure_hook_openblas_optarch_generic,
+ 'WRF': pre_configure_hook_wrf_aarch64,
}
diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml
new file mode 100644
index 0000000000..29301d6c77
--- /dev/null
+++ b/eessi-2023.06-eb-4.7.2-2021a.yml
@@ -0,0 +1 @@
+easyconfigs: []
diff --git a/init/eessi_defaults b/init/eessi_defaults
index f482cbc269..0143dc38ab 100644
--- a/init/eessi_defaults
+++ b/init/eessi_defaults
@@ -9,4 +9,4 @@
#
export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}"
-export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}"
+export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}"
diff --git a/load_easybuild_module.sh b/load_easybuild_module.sh
index 4ff2a3c37c..9897059f82 100755
--- a/load_easybuild_module.sh
+++ b/load_easybuild_module.sh
@@ -23,14 +23,14 @@ fi
EB_VERSION=${1}
# make sure that environment variables that we expect to be set are indeed set
-if [ -z "${TMPDIR}" ]; then
+if [ -z "${TMPDIR}" ]; then
echo "\$TMPDIR is not set" >&2
exit 2
fi
# ${EB} is used to specify which 'eb' command should be used;
# can potentially be more than just 'eb', for example when using 'eb --optarch=GENERIC'
-if [ -z "${EB}" ]; then
+if [ -z "${EB}" ]; then
echo "\$EB is not set" >&2
exit 2
fi