Skip to content

update eessi_container.sh script for software.eessi.io #390

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bedroge merged 2 commits into from
Nov 25, 2023
Merged

update eessi_container.sh script for software.eessi.io #390

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
50ec01a
update eessi_container.sh script for software.eessi.io
boegel Nov 25, 2023
d04f9bb
Merge branch '2023.06-software.eessi.io' into 2023.06-software.eessi.…
boegel Nov 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions .github/workflows/test_eessi_container_script.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
outfile=out_listrepos.txt
./eessi_container.sh --verbose --list-repos | tee ${outfile}
grep "EESSI-pilot" ${outfile}
grep "EESSI" ${outfile}

# test use of --list-repos with custom repos.cfg
elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
Expand All @@ -57,7 +57,7 @@ jobs:
echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
echo "repo_version = 20HT.TP" >> cfg/repos.cfg
./eessi_container.sh --verbose --list-repos | tee ${outfile}
grep "EESSI-pilot" ${outfile}
grep "EESSI" ${outfile}

export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
./eessi_container.sh --verbose --list-repos | tee ${outfile2}
Expand Down Expand Up @@ -90,15 +90,15 @@ jobs:
elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
outfile=out_readwrite.txt
fn="test_${RANDOM}.txt"
echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh
chmod u+x test_script.sh
export SINGULARITY_BIND="$PWD:/test"
./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}

tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
# note: must use '--access rw' again here, since touched file is in overlay upper dir
./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
grep "/cvmfs/software.eessi.io/${fn}$" $outfile

# test use of --resume
elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
Expand All @@ -120,12 +120,12 @@ jobs:
elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
outfile=out_save.txt
fn="test_${RANDOM}.txt"
test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
test_cmd="touch /cvmfs/software.eessi.io/${fn}"
./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
rm -f ${outfile}

./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
grep "/cvmfs/software.eessi.io/${fn}$" $outfile

tar tfv test-save.tar | grep "overlay-upper/${fn}"

Expand Down
24 changes: 12 additions & 12 deletions create_tarball.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ set -e
base_dir=$(dirname $(realpath $0))

if [ $# -ne 4 ]; then
echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <pilot version (example: 2021.03)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <version (example: 2023.06)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
exit 1
fi
eessi_tmpdir=$1
pilot_version=$2
eessi_version=$2
cpu_arch_subdir=$3
target_tgz=$4

Expand All @@ -20,15 +20,15 @@ os="linux"
source ${base_dir}/init/eessi_defaults
cvmfs_repo=${EESSI_CVMFS_REPO}

software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}"
software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
if [ ! -d ${software_dir} ]; then
echo "Software directory ${software_dir} does not exist?!" >&2
exit 2
fi

overlay_upper_dir="${eessi_tmpdir}/overlay-upper"

software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}"
software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
if [ ! -d ${software_dir_overlay} ]; then
echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2
exit 3
Expand All @@ -40,22 +40,22 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..
files_list=${tmpdir}/files.list.txt
module_files_list=${tmpdir}/module_files.list.txt

if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
# include Lmod cache and configuration file (lmodrc.lua),
# skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list}
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list}
fi
if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
# module files
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
# module symlinks
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
# module files and symlinks
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
| grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
>> ${module_files_list}
fi
if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
# installation directories but only those for which module files were created
# Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua'
# using EasyBuild's standard module naming scheme) match the name of the
Expand All @@ -64,7 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${modu
# installation directories), the procedure will likely not work.
for package_version in $(cat ${module_files_list}); do
echo "handling ${package_version}"
ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
| grep -v '/\.wh\.' >> ${files_list}
done
fi
Expand Down
40 changes: 22 additions & 18 deletions eessi_container.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ display_help() {
echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or"
echo " MODE==run (run a script or command) [default: shell]"
echo " -r | --repository CFG - configuration file or identifier defining the"
echo " repository to use [default: EESSI-pilot via"
echo " repository to use [default: EESSI via"
echo " default container, see --container]"
echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball,"
echo " where DIR points to a previously used tmp directory"
Expand Down Expand Up @@ -111,7 +111,7 @@ VERBOSE=0
STORAGE=
LIST_REPOS=0
MODE="shell"
REPOSITORY="EESSI-pilot"
REPOSITORY="EESSI"
RESUME=
SAVE=
HTTP_PROXY=${http_proxy:-}
Expand Down Expand Up @@ -194,7 +194,7 @@ set -- "${POSITIONAL_ARGS[@]}"

if [[ ${LIST_REPOS} -eq 1 ]]; then
echo "Listing available repositories with format 'name [source]':"
echo " EESSI-pilot [default]"
echo " EESSI [default]"
if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then
cfg_load ${EESSI_REPOS_CFG_FILE}
sections=$(cfg_sections)
Expand Down Expand Up @@ -226,7 +226,7 @@ fi

# TODO (arg -r|--repository) check if repository is known
# REPOSITORY_ERROR_EXITCODE
if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then
fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}"
fi

Expand Down Expand Up @@ -403,7 +403,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}"
# set up repository config (always create directory repos_cfg and populate it with info when
# arg -r|--repository is used)
mkdir -p ${EESSI_TMPDIR}/repos_cfg
if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then
if [[ "${REPOSITORY}" == "EESSI" ]]; then
# need to source defaults as late as possible (see other sourcing below)
source ${TOPDIR}/init/eessi_defaults

Expand All @@ -427,7 +427,7 @@ else
# map { local_filepath -> container_filepath }
#
# repo_name_domain is the domain part of the repo_name, e.g.,
# eessi-hpc.org for pilot.eessi-hpc.org
# eessi.io for software.eessi.io
#
# where config bundle includes the files (-> target location in container)
# - default.local -> /etc/cvmfs/default.local
Expand Down Expand Up @@ -479,7 +479,7 @@ else
target=${cfg_file_map[${src}]}
BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}"
done
export EESSI_PILOT_VERSION_OVERRIDE=${repo_version}
export EESSI_VERSION_OVERRIDE=${repo_version}
export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}"
# need to source defaults as late as possible (after *_OVERRIDEs)
source ${TOPDIR}/init/eessi_defaults
Expand Down Expand Up @@ -513,10 +513,14 @@ fi
# 4. set up vars and dirs specific to a scenario

declare -a EESSI_FUSE_MOUNTS=()

# always mount cvmfs-config repo (to get access to software.eessi.io)
EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch")

if [[ "${ACCESS}" == "ro" ]]; then
export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}"
export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}"

EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
export EESSI_FUSE_MOUNTS
fi

Expand All @@ -525,18 +529,18 @@ if [[ "${ACCESS}" == "rw" ]]; then
mkdir -p ${EESSI_TMPDIR}/overlay-work

# set environment variables for fuse mounts in Singularity container
export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"

EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}")
EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")

EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs"
EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
export EESSI_PILOT_WRITABLE_OVERLAY
EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}"
EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper"
EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work"
EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}"
export EESSI_WRITABLE_OVERLAY

EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}")
EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}")
export EESSI_FUSE_MOUNTS
fi

Expand Down