From a22ee425509b016fbaad5e0437087e8a2acd8696 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 19 Aug 2024 14:36:02 +0200 Subject: [PATCH 1/9] use upper directory of former 'rw' session in 'ro' mode --- eessi_container.sh | 71 +++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e404b7ee18..7f0c64ef27 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -254,7 +254,7 @@ if [[ ${LIST_REPOS} -eq 1 ]]; then default_label=", default" else default_label="" - fi + fi echo " ${cvmfs_repo} [CVMFS config repo${default_label}]" done for cfg_repo in "${!cfg_cvmfs_repos[@]}" @@ -323,7 +323,7 @@ do if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." # cvmfs_repo_name is actually a repository ID, use that to obtain - # the actual name from the EESSI_REPOS_CFG_FILE + # the actual name from the EESSI_REPOS_CFG_FILE cfg_repo_id=${cvmfs_repo_name} cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi @@ -595,11 +595,11 @@ do # that the necessary information for accessing a CVMFS repository is made # available inside the container if [[ -n "${cfg_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then - cfg_repo_id=${cvmfs_repo_name} + cfg_repo_id=${cvmfs_repo_name} - # obtain CVMFS repository name from section for the given ID + # obtain CVMFS repository name from section for the given ID cfg_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") - # derive domain part from (cfg_)repo_name (everything after first '.') + # derive domain part from (cfg_)repo_name (everything after first '.') repo_name_domain=${repo_name#*.} # cfg_cvmfs_repos is populated through reading the file pointed to by @@ -609,15 +609,15 @@ do # copy repos.cfg to job directory --> makes it easier to inspect the job cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. - # cfg file should include sections (one per CVMFS repository to be mounted) - # with each section containing the settings: - # - repo_name, - # - repo_version, - # - config_bundle, and - # - a map { filepath_in_bundle -> container_filepath } + # cfg file should include sections (one per CVMFS repository to be mounted) + # with each section containing the settings: + # - repo_name, + # - repo_version, + # - config_bundle, and + # - a map { filepath_in_bundle -> container_filepath } # - # The config_bundle includes the files which are mapped ('->') to a target - # location in container: + # The config_bundle includes the files which are mapped ('->') to a target + # location in container: # - default.local -> /etc/cvmfs/default.local # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf @@ -641,7 +641,7 @@ do # use information to set up dir ${EESSI_TMPDIR}/repos_cfg and define # BIND mounts # check if config_bundle exists, if so, unpack it into - # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error + # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error # if config_bundle is relative path (no '/' at start) prepend it with # EESSI_REPOS_CFG_DIR config_bundle_path= @@ -726,7 +726,7 @@ do if [[ ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." # cvmfs_repo_name is actually a repository ID, use that to obtain - # the actual name from the EESSI_REPOS_CFG_FILE + # the actual name from the EESSI_REPOS_CFG_FILE cfg_repo_id=${cvmfs_repo_name} cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi @@ -736,15 +736,46 @@ do # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) if [[ ${cvmfs_repo_access} == "ro" ]] ; then - export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" + # need to distinguish between basic "ro" access and "ro" after a "rw" session + if [[ -d ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ]]; then + # the overlay-upper directory is only created in a read-write-session, thus + # we are resuming from such a session here (otherwise there shouldn't be such + # directory yet as it is only created for read-write-sessions a bit further + # below); the overlay-upper directory can only exist because it is part of + # the ${RESUME} directory or tarball + # to be able to see the contents of the read-write session we have to mount + # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository + + # make the target CernVM-FS repository available under /cvmfs_ro + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + + # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs + EESSI_READONLY_OVERLAY="container:fuse-overlayfs" + # The contents of the previous session are available under + # ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}. + # Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper + # the left-most directory given for the lowerdir argument is put on top, + # and with no upperdir=... the whole overlayfs is made available read-only + EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}" + EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + export EESSI_READONLY_OVERLAY + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY_OVERLAY}") + export EESSI_FUSE_MOUNTS + else + # basic "ro" access that doesn't require any fuseoverlay-fs + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work - [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" @@ -762,7 +793,7 @@ do export EESSI_FUSE_MOUNTS else echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" - exit ${REPOSITORY_ERROR_EXITCODE} + exit ${REPOSITORY_ERROR_EXITCODE} fi # create repo_settings.sh file in ${EESSI_TMPDIR}/${cvmfs_repo_name} to store # (intention is that the file could be just sourced to obtain the settings) From d75784cd858aade47523173c60340a19b35692c9 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 16:49:08 +0200 Subject: [PATCH 2/9] Test if we can find and use a module in the test step that has been build in this same PR --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 ++- test_suite.sh | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index a22b78718f..6996e6f712 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -6,4 +6,5 @@ easyconfigs: from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - gnuplot-5.4.6-GCCcore-12.2.0.eb - h5py-3.8.0-foss-2022b.eb - - MDAnalysis-2.4.2-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - patchelf-0.17.2-GCCcore-12.2.0.eb diff --git a/test_suite.sh b/test_suite.sh index 6e73fbd87c..55fb2453a4 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -79,6 +79,11 @@ export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py source $TOPDIR/init/bash +# DEBUG: only to test PR674, this code should never be merged +module av patchelf/0.17.2-GCCcore-12.2.0 +module load patchelf/0.17.2-GCCcore-12.2.0 +patchelf --help + # Load the ReFrame module # Currently, we load the default version. Maybe we should somehow make this configurable in the future? module load ReFrame From c1939e48459a4ea087481a415a970e38253cf522 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 16:54:02 +0200 Subject: [PATCH 3/9] Try to also write something in the CVMFS repo --- test_suite.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test_suite.sh b/test_suite.sh index 55fb2453a4..511272b60f 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -83,6 +83,7 @@ source $TOPDIR/init/bash module av patchelf/0.17.2-GCCcore-12.2.0 module load patchelf/0.17.2-GCCcore-12.2.0 patchelf --help +touch /cvmfs/software.eessi.io/foo # Load the ReFrame module # Currently, we load the default version. Maybe we should somehow make this configurable in the future? From 839c474c916e7fa196b45fef59eb52645a1ac48d Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 19 Aug 2024 17:07:48 +0200 Subject: [PATCH 4/9] Ignore cache --- test_suite.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 511272b60f..d29c1be632 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -81,7 +81,8 @@ source $TOPDIR/init/bash # DEBUG: only to test PR674, this code should never be merged module av patchelf/0.17.2-GCCcore-12.2.0 -module load patchelf/0.17.2-GCCcore-12.2.0 +module --ignore_cache load patchelf/0.17.2-GCCcore-12.2.0 +which patchelf patchelf --help touch /cvmfs/software.eessi.io/foo @@ -200,7 +201,9 @@ fi # Run all tests echo "Running tests: reframe ${REFRAME_ARGS} --run" -reframe ${REFRAME_ARGS} --run +# reframe ${REFRAME_ARGS} --run +# Make this run faster, we are not interestied in the test step here... +reframe ${REFRAME_ARGS} --list reframe_exit_code=$? if [[ ${reframe_exit_code} -eq 0 ]]; then echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run." From 62fb7b9960a17a12a5cf6d0fe3fba8b97a06a9b2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 09:55:11 +0200 Subject: [PATCH 5/9] More verbose, print ebrootpatchelf and command -v patchelf to see if we can't figure out where the patchelf is coming from --- eessi_container.sh | 2 ++ test_suite.sh | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 7f0c64ef27..5fadbb69ca 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -745,6 +745,8 @@ do # the ${RESUME} directory or tarball # to be able to see the contents of the read-write session we have to mount # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository + echo "Overlay-upper exists, so assuming we are resuming from a previous step." + echo "Since repo-access mode 'ro' was requested, we are mounting it as lowerdir to make it read-only." # make the target CernVM-FS repository available under /cvmfs_ro export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" diff --git a/test_suite.sh b/test_suite.sh index d29c1be632..50662e3112 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -82,7 +82,8 @@ source $TOPDIR/init/bash # DEBUG: only to test PR674, this code should never be merged module av patchelf/0.17.2-GCCcore-12.2.0 module --ignore_cache load patchelf/0.17.2-GCCcore-12.2.0 -which patchelf +command patchelf +echo $EBROOTPATCHELF patchelf --help touch /cvmfs/software.eessi.io/foo From 76f84d8e98ed377438928e7b42b99b22f7889f17 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 10:19:59 +0200 Subject: [PATCH 6/9] Extra output for debugging --- bot/test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/test.sh b/bot/test.sh index d3f3630ea8..bee6e0f663 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -219,6 +219,9 @@ fi # create tmp file for output of build step test_outerr=$(mktemp test.outerr.XXXX) +echo "Listing known repos" +./eessi_container.sh --list-repos + echo "Executing command to test software:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}" echo " -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}" From 057612418a5c83f5a67ede8d6456ae7173296ebe Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 10:35:50 +0200 Subject: [PATCH 7/9] More debugging output --- test_suite.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 50662e3112..b68e45e516 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -82,9 +82,13 @@ source $TOPDIR/init/bash # DEBUG: only to test PR674, this code should never be merged module av patchelf/0.17.2-GCCcore-12.2.0 module --ignore_cache load patchelf/0.17.2-GCCcore-12.2.0 -command patchelf -echo $EBROOTPATCHELF +echo "Command -v patchelf" +command -v patchelf +echo "EBROOTPATCHELF: $EBROOTPATCHELF" +echo "patchelf --help:" patchelf --help +echo "ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/modules/all/patchelf/" +ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/modules/all/patchelf/ touch /cvmfs/software.eessi.io/foo # Load the ReFrame module From 603e7ae8faf1ce89a106450315c16407cc0389a6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 10:36:47 +0200 Subject: [PATCH 8/9] Make it work for any arch --- test_suite.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index b68e45e516..3f23532dcd 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -87,8 +87,8 @@ command -v patchelf echo "EBROOTPATCHELF: $EBROOTPATCHELF" echo "patchelf --help:" patchelf --help -echo "ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/modules/all/patchelf/" -ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/modules/all/patchelf/ +echo "ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all/patchelf/" +ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all/patchelf/ touch /cvmfs/software.eessi.io/foo # Load the ReFrame module From e2af8587485bc447f18a48509802497260045103 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 20 Aug 2024 10:44:18 +0200 Subject: [PATCH 9/9] More verbosity still --- test_suite.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 3f23532dcd..4015212dd1 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -80,13 +80,17 @@ export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py source $TOPDIR/init/bash # DEBUG: only to test PR674, this code should never be merged +echo "module av patchelf/0.17.2-GCCcore-12.2.0" module av patchelf/0.17.2-GCCcore-12.2.0 +echo "module load patchelf/0.17.2-GCCcore-12.2.0" +module load patchelf/0.17.2-GCCcore-12.2.0 +echo "module --ignore_cache load patchelf/0.17.2-GCCcore-12.2.0" module --ignore_cache load patchelf/0.17.2-GCCcore-12.2.0 echo "Command -v patchelf" command -v patchelf echo "EBROOTPATCHELF: $EBROOTPATCHELF" -echo "patchelf --help:" -patchelf --help +echo "patchelf --version" +patchelf --version echo "ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all/patchelf/" ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all/patchelf/ touch /cvmfs/software.eessi.io/foo