From 9766267f475b1f266923b6c03b545d10caabd42a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 16 Dec 2021 19:26:11 +0100 Subject: [PATCH 1/9] also install SciPy-bundle-2021.05-foss-2021a.eb into 2021.12 pilot --- EESSI-pilot-install-software.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 10c57c6d13..4cbf0d3b8e 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -326,6 +326,24 @@ fail_msg="Installation of IPython failed, that's unexpected..." $EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r check_exit_code $? "${ok_msg}" "${fail_msg}" +echo ">> Installing EasyBuild 4.5.1..." +ok_msg="EasyBuild v4.5.1 installed" +fail_msg="EasyBuild v4.5.1 failed to install" +$EB --from-pr 14545 +check_exit_code $? "${ok_msg}" "${fail_msg}" + +LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 +check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" + +echo ">> Installing SciPy-bundle with foss/2021a..." +ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" +fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." +$EB SciPy-bundle-2021.05-foss-2021a.eb -M +$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot +$EB SciPy-bundle-2021.05-foss-2021a.eb -r --buildpath /dev/shm/$USER/easybuild_build +check_exit_code $? "${ok_msg}" "${fail_msg}" + echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" if [ ! -f $LMOD_RC ]; then From 596ab7adc3ac5d7fe045b4be6fc3b4b6aad4f840 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 16 Dec 2021 19:41:37 +0100 Subject: [PATCH 2/9] update hooks to add --disable-psm3 when building libfabric on x86_64 without AVX support --- eb_hooks.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 126d89ca46..7eb7627643 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -4,7 +4,8 @@ from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option -from easybuild.tools.systemtools import POWER, get_cpu_architecture +from easybuild.tools.systemtools import POWER, X86_64, get_cpu_architecture, get_cpu_features +from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -52,6 +53,13 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) +def pre_configure_hook(self, *args, **kwargs): + """Main pre-configure hook: trigger custom functions based on software name.""" + + if self.name in PRE_CONFIGURE_HOOKS: + PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) + + def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -114,12 +122,25 @@ def fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") +def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): + """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64.""" + if self.name == 'libfabric': + if get_cpu_architecture() == X86_64: + generic = build_option('optarch') == OPTARCH_GENERIC + no_avx = 'avx' not in get_cpu_features() + if generic or no_avx: + self.cfg.update('configopts', '--disable-psm3') + print_msg("Using custom configure options for %s: %s", self.name, self.cfg['configopts']) + else: + raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!") + + def ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr')) - print_msg("Using custom configure option for %s: %s", ec.name, ec['configopts']) + print_msg("Using custom configure options for %s: %s", ec.name, ec['configopts']) else: raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!") @@ -129,3 +150,7 @@ def ucx_eprefix(ec, eprefix): 'fontconfig': fontconfig_add_fonts, 'UCX': ucx_eprefix, } + +PRE_CONFIGURE_HOOKS = { + 'libfabric': libfabric_disable_psm3_x86_64_generic, +} From 7adc0764864f45b85b1e0feeb27c90cd86fc736e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Jan 2022 17:07:41 +0100 Subject: [PATCH 3/9] reorder software alphabetically in easystack file for 2021.12 --- eessi-2021.12.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml index 0d060f55ac..cc1519e30b 100644 --- a/eessi-2021.12.yml +++ b/eessi-2021.12.yml @@ -1,10 +1,8 @@ software: - R-bundle-Bioconductor: + code-server: toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 + SYSTEM: + versions: '3.7.3' GROMACS: toolchains: foss-2020a: @@ -31,19 +29,21 @@ software: toolchains: foss-2020a: versions: ['6.6'] - TensorFlow: + R-bundle-Bioconductor: toolchains: foss-2020a: versions: - '2.3.1': - versionsuffix: -Python-3.8.2 + '3.11': + versionsuffix: -R-4.0.0 RStudio-Server: toolchains: foss-2020a: versions: '1.3.1093': versionsuffix: -Java-11-R-4.0.0 - code-server: + TensorFlow: toolchains: - SYSTEM: - versions: '3.7.3' + foss-2020a: + versions: + '2.3.1': + versionsuffix: -Python-3.8.2 From 1e3d482ed47a4defd819d7826f29678dfcf4f8d3 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 6 Jan 2022 17:12:22 +0100 Subject: [PATCH 4/9] add SciPy-bundle with foss/2021a to easystack file for 2021.12 EESSI pilot --- eessi-2021.12.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml index cc1519e30b..d2e1ccb1ba 100644 --- a/eessi-2021.12.yml +++ b/eessi-2021.12.yml @@ -41,6 +41,10 @@ software: versions: '1.3.1093': versionsuffix: -Java-11-R-4.0.0 + SciPy-bundle: + toolchains: + foss-2021a: + versions: ['2021.05'] TensorFlow: toolchains: foss-2020a: From e5d67b2c16650c40551a6513ed64d25053e14613 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 7 Jan 2022 15:59:49 +0100 Subject: [PATCH 5/9] use enhanced Perl easyblock to fix problem with excessive long shebang line in Perl scripts --- EESSI-pilot-install-software.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4cbf0d3b8e..be699069d6 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -339,7 +339,14 @@ echo ">> Installing SciPy-bundle with foss/2021a..." ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." $EB SciPy-bundle-2021.05-foss-2021a.eb -M +# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 +# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set +# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 +# that includes patch to fix bootstrap problem when using alternate sysroot $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot $EB SciPy-bundle-2021.05-foss-2021a.eb -r --buildpath /dev/shm/$USER/easybuild_build check_exit_code $? "${ok_msg}" "${fail_msg}" From b80be7a8f723323fb7a3662b0a4ef856f2bb573d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 7 Nov 2022 19:52:34 +0100 Subject: [PATCH 6/9] use merged PRs to fix installation of GCCcore/10.3.0 with recent kernel headers + workaround for compiler bug affecting OpenBLAS on POWER --- EESSI-pilot-install-software.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 193f5600a1..7fdcbde6b6 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -344,7 +344,9 @@ check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5. echo ">> Installing SciPy-bundle with foss/2021a..." ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -$EB SciPy-bundle-2021.05-foss-2021a.eb -M +# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 +# which includes patch to fix installation with recent Linux kernel headers +$EB --from-pr 14454 GCCcore-10.3.0.eb --robot # use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 # to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) $EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 @@ -354,6 +356,10 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 # use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 # that includes patch to fix bootstrap problem when using alternate sysroot $EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot +# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 +# which includes a patch to fix installation on POWER +$EB --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot + $EB SciPy-bundle-2021.05-foss-2021a.eb -r --buildpath /dev/shm/$USER/easybuild_build check_exit_code $? "${ok_msg}" "${fail_msg}" From 65812efb82578b5cc584e0bd3ab9b16f4e00681f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 7 Nov 2022 20:06:52 +0100 Subject: [PATCH 7/9] use updated EasyBuildMeta easyblock to install EasyBuild 4.5.1 --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f23d60a328..983ff9d1bd 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -359,7 +359,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Installing EasyBuild 4.5.1..." ok_msg="EasyBuild v4.5.1 installed" fail_msg="EasyBuild v4.5.1 failed to install" -$EB --from-pr 14545 +$EB --from-pr 14545 --include-easyblocks-from-pr 2805 check_exit_code $? "${ok_msg}" "${fail_msg}" LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 From 83b283c66720942371f88ce987b932f94d26c8c5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 8 Nov 2022 19:34:17 +0100 Subject: [PATCH 8/9] fix alphabetical sorting in eessi-2021.12.yml --- eessi-2021.12.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml index 102349551b..977c0f9804 100644 --- a/eessi-2021.12.yml +++ b/eessi-2021.12.yml @@ -17,6 +17,10 @@ software: versions: '0.21.3': versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 + Nextflow: + toolchains: + SYSTEM: + versions: '22.10.1' OpenFOAM: toolchains: foss-2020a: @@ -57,7 +61,3 @@ software: versions: '3.9.1.1': versionsuffix: -dmpar - Nextflow: - toolchains: - SYSTEM: - versions: '22.10.1' From edaae9db896a57d66babb2e4a7c516b6b1db0578 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 14 Dec 2022 15:44:51 +0100 Subject: [PATCH 9/9] Whoops --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 391cf8813b..df7742f999 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -163,7 +163,7 @@ def metabat_preconfigure(self, *args, **kwargs): regex = re.compile(r"\$EBROOTZLIB/lib/libz.a") self.cfg['configopts'] = regex.sub('$EPREFIX/usr/lib64/libz.so', configopts) else: - raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!" + raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!") def wrf_preconfigure(self, *args, **kwargs):