From d3ceed500c86852200e475ed74f2d95a3fddefcd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 13 Jan 2023 21:45:01 +0100 Subject: [PATCH 1/4] flesh out script for checking on missing installations + test it in CI --- .github/workflows/test_eessi.yml | 41 ++++++++++++++++++++++++++++++++ EESSI-pilot-install-software.sh | 10 ++------ check_missing_installations.sh | 21 ++++++++++++++++ 3 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/test_eessi.yml create mode 100755 check_missing_installations.sh diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml new file mode 100644 index 0000000000..2b57eccc90 --- /dev/null +++ b/.github/workflows/test_eessi.yml @@ -0,0 +1,41 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests relying on having EESSI pilot repo mounted +on: [push, pull_request, workflow_dispatch] +jobs: + eessi_pilot_repo: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2021.12 + EESSI_SOFTWARE_SUBDIR: + - aarch64/generic + - aarch64/graviton2 + - aarch64/graviton3 + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/generic + steps: + - name: Check out software-layer repository + uses: actions/checkout@v2 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@main + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: pilot.eessi-hpc.org + + - name: Test check_missing_installations.sh script + run: | + source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + ./check_missing_installations.sh diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index d03c2bdce7..57c078f511 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -118,7 +118,7 @@ else fi echo ">> Configuring EasyBuild..." -source configure_easybuild +source $TOPDIR/configure_easybuild echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded @@ -421,13 +421,7 @@ fi $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} -echo ">> Checking for missing installations..." -ok_msg="No missing installations, party time!" -fail_msg="On no, some installations are still missing, how did that happen?!" -eb_missing_out=$TMPDIR/eb_missing.out -$EB --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing --robot $EASYBUILD_PREFIX/ebfiles_repo | tee ${eb_missing_out} -grep "No missing modules" ${eb_missing_out} > /dev/null -check_exit_code $? "${ok_msg}" "${fail_msg}" +$TOPDIR/check_missing_installations.sh echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} diff --git a/check_missing_installations.sh b/check_missing_installations.sh new file mode 100755 index 0000000000..9675397164 --- /dev/null +++ b/check_missing_installations.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Script to check for missing installations in EESSI pilot software stack (version 2021.12) + +TOPDIR=$(dirname $(realpath $0)) + +TMPDIR=$(mktemp -d) + +source $TOPDIR/utils.sh + +source $TOPDIR/configure_easybuild + +echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." +ok_msg="No missing installations, party time!" +fail_msg="On no, some installations are still missing, how did that happen?!" +eb_missing_out=$TMPDIR/eb_missing.out +# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used +# PR #16531: Nextflow-22.10.1.eb +${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} +grep "No missing modules" ${eb_missing_out} > /dev/null +check_exit_code $? "${ok_msg}" "${fail_msg}" From cc15fd577a811529941ef80ae89797c685d95686 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 13 Jan 2023 22:17:53 +0100 Subject: [PATCH 2/4] fix issues in CI workflow raised by OSSF scorecard --- .github/workflows/test_eessi.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 2b57eccc90..92b1f71cad 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -1,6 +1,8 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests relying on having EESSI pilot repo mounted on: [push, pull_request, workflow_dispatch] +permissions: + contents: read # to fetch code (actions/checkout) jobs: eessi_pilot_repo: runs-on: ubuntu-20.04 @@ -20,10 +22,10 @@ jobs: - x86_64/generic steps: - name: Check out software-layer repository - uses: actions/checkout@v2 + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@main + uses: cvmfs-contrib/github-action-cvmfs@d4641d0d591c9a5c3be23835ced2fb648b44c04b # v3.1 with: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT From a7909eee1a768c4f7a3723386eb561a71716f35d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 14 Jan 2023 08:20:50 +0100 Subject: [PATCH 3/4] add license header + reuse $TMPDIR if set + make sure that $EESSI_PILOT_VERSION is set in check_missing_installations.sh --- check_missing_installations.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 9675397164..cfddb0ee14 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,10 +1,21 @@ #!/bin/bash # # Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# +# author: Kenneth Hoste (@boegel) +# +# license: GPLv2 +# TOPDIR=$(dirname $(realpath $0)) -TMPDIR=$(mktemp -d) +if [ -z ${EESSI_PILOT_VERSION} ]; then + echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 + exit 1 +fi + +# reuse existing $TMPDIR if set, create a new tmpdir if $TMPDIR is unset +TMPDIR=${TMPDIR:-$(mktemp -d)} source $TOPDIR/utils.sh From 5c940744415fa9688bbd4a9e64c2ae146469f348 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 14 Jan 2023 14:40:43 +0100 Subject: [PATCH 4/4] rename $TMPDIR to $LOCAL_TMPDIR in check_missing_installations.sh, since $TMPDIR is a common environment variable (picked up by mktemp, etc.) --- check_missing_installations.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index cfddb0ee14..30f9cc6ff7 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -14,8 +14,7 @@ if [ -z ${EESSI_PILOT_VERSION} ]; then exit 1 fi -# reuse existing $TMPDIR if set, create a new tmpdir if $TMPDIR is unset -TMPDIR=${TMPDIR:-$(mktemp -d)} +LOCAL_TMPDIR=$(mktemp -d) source $TOPDIR/utils.sh @@ -24,7 +23,7 @@ source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." ok_msg="No missing installations, party time!" fail_msg="On no, some installations are still missing, how did that happen?!" -eb_missing_out=$TMPDIR/eb_missing.out +eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}