From f59bfdededf7b56bc95dc64f19021a3770d8ac61 Mon Sep 17 00:00:00 2001 From: trz42 Date: Tue, 7 Mar 2023 15:33:24 +0100 Subject: [PATCH 01/12] fix check for missing installations --- check_missing_installations.sh | 20 ++++++++++++++++++-- scripts/utils.sh | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index e4c75aebd8..45f276dee1 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -27,5 +27,21 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} -grep "No missing modules" ${eb_missing_out} > /dev/null -check_exit_code $? "${ok_msg}" "${fail_msg}" + +# the above assesses the installed software for each easyconfig provided in +# the easystack file and then print messages such as +# `No missing modules!` +# or +# `2 out of 3 required modules missing:` +# depending on the result of the assessment. Hence, we need to check if the +# output does not contain any line with ` required modules missing:` + +grep " required modules missing:" ${eb_missing_out} > /dev/null + +# we need to process the result (from finding `No missing modules` to NOT finding +# ` required modules missing:` and no other error happened) +# +# if grep returns 1 (` required modules missing:` was NOT found), we set +# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 +[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" diff --git a/scripts/utils.sh b/scripts/utils.sh index d0da95e87f..1f46ae69b4 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -28,7 +28,7 @@ function check_exit_code { if [[ $ec -eq 0 ]]; then echo_green "${ok_msg}" else - fatal_error "${fail_msg}" + echo_red "${fail_msg}" fi } From 7bb7c1de90b2bfb47a11124702d6dd968531267a Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:07:04 +0100 Subject: [PATCH 02/12] add test with missing package --- .github/workflows/test_eessi.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 92b1f71cad..1735bb77ed 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -40,4 +40,13 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" + echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml + ./check_missing_installations.sh + [[ $? -eq 0 ]] && exit 1 || exit 0 From 68a068b64ea5af8fad5a69ff812a5942cacd7167 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:25:04 +0100 Subject: [PATCH 03/12] separated test cases + updated evaluation of test result --- .github/workflows/test_eessi.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 1735bb77ed..073edaf4e8 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -42,11 +42,26 @@ jobs: env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + + - name: Test check_missing_installations.sh with missing package + run: | + source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml - ./check_missing_installations.sh - [[ $? -eq 0 ]] && exit 1 || exit 0 + if ./check_missing_installations.sh; then + echo "captured missing package; test PASSED" + exit 0 + else + echo "did NOT capture missing package; test FAILED" + exit 1 + fi From 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:30:17 +0100 Subject: [PATCH 04/12] fix small bug in env var name --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 073edaf4e8..f3bf2e7ab1 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -38,7 +38,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh @@ -50,7 +50,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 094e82df3b0de5a195d99fa914674024aa2e8145 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:35:29 +0100 Subject: [PATCH 05/12] set software subdir override before sourcing init script --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index f3bf2e7ab1..d0b530e2e4 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -33,24 +33,24 @@ jobs: - name: Test check_missing_installations.sh script run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - name: Test check_missing_installations.sh with missing package run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 60cf7e2715b7231810479a3fd91d20499c9b7670 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:42:01 +0100 Subject: [PATCH 06/12] Revert "set software subdir override before sourcing init script" This reverts commit 094e82df3b0de5a195d99fa914674024aa2e8145. --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index d0b530e2e4..f3bf2e7ab1 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -33,24 +33,24 @@ jobs: - name: Test check_missing_installations.sh script run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - name: Test check_missing_installations.sh with missing package run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From 6f471520b63b0cee6b9d94818e7f42938dbc1bf8 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:42:51 +0100 Subject: [PATCH 07/12] Revert "fix small bug in env var name" This reverts commit 72185ba91dde02c1852d5b59ab690ccb0d5d1ceb. --- .github/workflows/test_eessi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index f3bf2e7ab1..073edaf4e8 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -38,7 +38,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh @@ -50,7 +50,7 @@ jobs: eb --version export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR}} + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml From a15676ddac70b5cb18d82625a9a048d59199a41b Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 11:48:13 +0100 Subject: [PATCH 08/12] tweaking output of test --- .github/workflows/test_eessi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 073edaf4e8..23b30c0063 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -43,7 +43,7 @@ jobs: echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh - - name: Test check_missing_installations.sh with missing package + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash module load EasyBuild @@ -52,12 +52,12 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package" + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml - tail -n 8 eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml if ./check_missing_installations.sh; then echo "captured missing package; test PASSED" exit 0 From 99231e84eade092ae45fb5fa725d752eb04edf92 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 00:07:09 +0100 Subject: [PATCH 09/12] if exit_code is not zero, use fatal_error (as it was) --- scripts/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/utils.sh b/scripts/utils.sh index 1f46ae69b4..d0da95e87f 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -28,7 +28,7 @@ function check_exit_code { if [[ $ec -eq 0 ]]; then echo_green "${ok_msg}" else - echo_red "${fail_msg}" + fatal_error "${fail_msg}" fi } From b94bd111a3dcf422e7e78fa58bdbf52a74415fb2 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 00:52:43 +0100 Subject: [PATCH 10/12] fix bug in test logic --- .github/workflows/test_eessi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 23b30c0063..ce58fc3a97 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -59,9 +59,9 @@ jobs: echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml if ./check_missing_installations.sh; then - echo "captured missing package; test PASSED" - exit 0 - else echo "did NOT capture missing package; test FAILED" exit 1 + else + echo "captured missing package; test PASSED" + exit 0 fi From 5e4bea9b71a37a09359def990fd12a1c01a5bf1c Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 01:03:20 +0100 Subject: [PATCH 11/12] added explanation for test cases (exit code vs boolean logic) --- .github/workflows/test_eessi.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index ce58fc3a97..04195dd619 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -58,6 +58,11 @@ jobs: echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + # note, check_missing_installations.sh exits 1 if a package was + # missing, which is intepreted as false (exit code based, not + # boolean logic), hence when the script exits 0 if no package was + # missing it is interpreted as true, thus the test did not capture + # the missing package if ./check_missing_installations.sh; then echo "did NOT capture missing package; test FAILED" exit 1 From a4c8e6471bad5ce80df906c5126f5a02a06c55a0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 11 Apr 2023 09:35:44 +0200 Subject: [PATCH 12/12] removing comment to address review comment --- check_missing_installations.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 45f276dee1..926f475903 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -38,9 +38,6 @@ ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experim grep " required modules missing:" ${eb_missing_out} > /dev/null -# we need to process the result (from finding `No missing modules` to NOT finding -# ` required modules missing:` and no other error happened) -# # if grep returns 1 (` required modules missing:` was NOT found), we set # MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 [[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1