From e8c2d0636e087d4e565d5d5993d2894fa21da6b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Mon, 17 Jun 2024 15:10:28 +0200 Subject: [PATCH 01/11] first steps to support multiple CVMFS repositories --- eessi_container.sh | 99 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 77 insertions(+), 22 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index fb14e2118f..9b1ab69a3b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -48,7 +48,11 @@ HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) +# we use an associative array for storing sets of settings per CVMFS repository +declare -A cvmfs_repo_settings + # CernVM-FS settings +# TODO may need to put them into repository specific map CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" @@ -89,8 +93,9 @@ display_help() { echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI via" - echo " default container, see --container]" + echo " repository to use; can be given multiple times" + echo " [default: software.eessi.io via CVMFS config available" + echo " via default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" echo " (check for output 'Using DIR as tmp ...' of a previous" @@ -123,7 +128,7 @@ STORAGE= LIST_REPOS=0 MODE="shell" SETUP_NVIDIA=0 -REPOSITORY="EESSI" +REPOSITORIES=() RESUME= SAVE= HTTP_PROXY=${http_proxy:-} @@ -179,7 +184,7 @@ while [[ $# -gt 0 ]]; do shift 2 ;; -r|--repository) - REPOSITORY="$2" + REPOSITORIES+=("$2") shift 2 ;; -s|--save) @@ -221,22 +226,50 @@ done set -- "${POSITIONAL_ARGS[@]}" +# define a list of CVMFS repositories that are accessible via the +# CVMFS config repository which is always mounted +declare -A eessi_cvmfs_repos=(["dev.eessi.io"]="extra", ["riscv.eessi.io"]="extra", ["software.eessi.io"]="default") +eessi_default_cvmfs_repo="software.eessi.io" + +# if REPOSITORIES is empty add default repository given above +if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then + REPOSITORIES+=(${eessi_default_cvmfs_repo}) +fi + +# define a list of CVMFS repositories that are accessible via the +# configuration file provided via $EESSI_REPOS_CFG_FILE +declare -A cfg_cvmfs_repos=() +if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + cfg_cvmfs_repos[${repo_id}]=${EESSI_REPOS_CFG_FILE} + done <<< "${sections}" +fi + if [[ ${LIST_REPOS} -eq 1 ]]; then - echo "Listing available repositories with format 'name [source]':" - echo " EESSI [default]" - if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then - cfg_load ${EESSI_REPOS_CFG_FILE} - sections=$(cfg_sections) - while IFS= read -r repo_id - do - echo " ${repo_id} [${EESSI_REPOS_CFG_FILE}]" - done <<< "${sections}" - fi + echo "Listing available repositories with format 'name [source[, 'default']]'." + echo "Note, without argument '--repository' the one labeled 'default' will be mounted." + for cvmfs_repo in "${!eessi_cvmfs_repos[@]}" + do + if [[ ${eessi_cvmfs_repos[${cvmfs_repo}]} == "default" ]] ; then + default_label=", default" + else + default_label="" + fi + echo " ${cvmfs_repo} [CVMFS config repo${default_label}]" + done + for cfg_repo in "${!cfg_cvmfs_repos[@]}" + do + echo " ${cfg_repo} [${cfg_cvmfs_repos[$cfg_repo]}]" + done exit 0 fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported +# TODO use the value as global setting, suffix to --repository can specify an access mode per repository if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -260,10 +293,16 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then fi fi -# TODO (arg -r|--repository) check if repository is known +# TODO (arg -r|--repository) check if all explicitly listed repositories are known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then - fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" +if [[ ${#REPOSITORIES[@]} -ne 0 ]] ; then + # iterate over entries in REPOSITORIES and check if they are known + for cvmfs_repo in "${REPOSITORIES[@]}" + do + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo}]} ]]; then + fatal_error "The repository '${cvmfs_repo}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" + fi + done fi # TODO (arg -u|--resume) check if it exists, if user has read permission, @@ -337,22 +376,36 @@ if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then fi # 3. set up common vars and directories +# TODO change to be able to support multiple CVMFS repositories # directory structure should be: # ${EESSI_HOST_STORAGE} # |-singularity_cache -# |-${CVMFS_VAR_LIB} -# |-${CVMFS_VAR_RUN} -# |-overlay-upper -# |-overlay-work # |-home # |-repos_cfg -# |-opt-eessi (unless otherwise specificed for host_injections) +# |-CVMFS_REPO_1 +# | |-repo_settings (name, access_mode, host_injections) +# | |-${CVMFS_VAR_LIB} +# | |-${CVMFS_VAR_RUN} +# | |-overlay-upper +# | |-overlay-work +# | |-opt-eessi (unless otherwise specificed for host_injections) +# |-CVMFS_REPO_n +# |-repo_settings (name, access_mode, host_injections) +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} +# |-overlay-upper +# |-overlay-work +# |-opt-eessi (unless otherwise specificed for host_injections) # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +# TODO make this specific to repository +# TODO move this code to when we already know which repositories we want to access +# actually we should know this already here, but we should rather move this to +# where repository args are being processed # Set host_injections directory and ensure it is a writable directory (if user provided) if [ -z ${USER_HOST_INJECTIONS+x} ]; then # Not set, so use our default @@ -486,6 +539,8 @@ if [[ ${FAKEROOT} -eq 1 ]]; then ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") fi +exit 0; # CONTINUE HERE +# TODO iterate over repositories in array REPOSITORIES # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg From c7a1ca2fbbb736b3eea2242c7143aaa3be063fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Tue, 18 Jun 2024 15:05:17 +0200 Subject: [PATCH 02/11] further changes to support multiple repositories --- eessi_container.sh | 248 +++++++++++++++++++++++++++------------------ 1 file changed, 148 insertions(+), 100 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9b1ab69a3b..d561d37792 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -228,16 +228,14 @@ set -- "${POSITIONAL_ARGS[@]}" # define a list of CVMFS repositories that are accessible via the # CVMFS config repository which is always mounted +# TODO instead of hard-coding the 'extra' and 'default' repositories here one +# could have another script in the GitHub and/or CVMFS repository which +# provides this "configuration" declare -A eessi_cvmfs_repos=(["dev.eessi.io"]="extra", ["riscv.eessi.io"]="extra", ["software.eessi.io"]="default") -eessi_default_cvmfs_repo="software.eessi.io" - -# if REPOSITORIES is empty add default repository given above -if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then - REPOSITORIES+=(${eessi_default_cvmfs_repo}) -fi +eessi_default_cvmfs_repo="software.eessi.io,access=${ACCESS}" # define a list of CVMFS repositories that are accessible via the -# configuration file provided via $EESSI_REPOS_CFG_FILE +# configuration file provided via $EESSI_REPOS_CFG_FILE declare -A cfg_cvmfs_repos=() if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then cfg_load ${EESSI_REPOS_CFG_FILE} @@ -267,6 +265,11 @@ if [[ ${LIST_REPOS} -eq 1 ]]; then exit 0 fi +# if REPOSITORIES is empty add default repository given above +if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then + REPOSITORIES+=(${eessi_default_cvmfs_repo}) +fi + # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported # TODO use the value as global setting, suffix to --repository can specify an access mode per repository @@ -295,15 +298,45 @@ fi # TODO (arg -r|--repository) check if all explicitly listed repositories are known # REPOSITORY_ERROR_EXITCODE -if [[ ${#REPOSITORIES[@]} -ne 0 ]] ; then - # iterate over entries in REPOSITORIES and check if they are known - for cvmfs_repo in "${REPOSITORIES[@]}" - do - if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo}]} ]]; then - fatal_error "The repository '${cvmfs_repo}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" +# iterate over entries in REPOSITORIES and check if they are known +for cvmfs_repo in "${REPOSITORIES[@]}" +do + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + else + cvmfs_repo_name="${cvmfs_repo}" + fi + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + fatal_error "The repository '${cvmfs_repo_name}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" + fi +done + +# make sure each repository is only listed once +declare -A listed_repos=() +for cvmfs_repo in "${REPOSITORIES[@]}" +do + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode + echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" + # if cvmfs_repo_name is not in eessi_cvmfs_repos, assume it's in cfg_cvmfs_repos + # and obtain actual repo_name from config + cfg_repo_id='' + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi + if [[ -n "${listed_repos[${cvmfs_repo_name}]}" ]] ; then + via_cfg="" + if [[ -n "${cfg_repo_id}" ]] ; then + via_cfg=" (via repository ID '${cfg_repo_id}')" fi - done -fi + fatal_error "CVMFS repository '${cvmfs_repo_name}'${via_cfg} listed multiple times" + fi + listed_repos+=([${cvmfs_repo_name}]=true) +done # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -382,17 +415,15 @@ fi # |-singularity_cache # |-home # |-repos_cfg +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} # |-CVMFS_REPO_1 # | |-repo_settings (name, access_mode, host_injections) -# | |-${CVMFS_VAR_LIB} -# | |-${CVMFS_VAR_RUN} # | |-overlay-upper # | |-overlay-work # | |-opt-eessi (unless otherwise specificed for host_injections) # |-CVMFS_REPO_n # |-repo_settings (name, access_mode, host_injections) -# |-${CVMFS_VAR_LIB} -# |-${CVMFS_VAR_RUN} # |-overlay-upper # |-overlay-work # |-opt-eessi (unless otherwise specificed for host_injections) @@ -402,7 +433,7 @@ EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" -# TODO make this specific to repository +# TODO make this specific to repository? # TODO move this code to when we already know which repositories we want to access # actually we should know this already here, but we should rather move this to # where repository args are being processed @@ -481,6 +512,7 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" # set env vars and create directories for CernVM-FS +# TODO need to use separate values for separate repos? EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} @@ -539,92 +571,106 @@ if [[ ${FAKEROOT} -eq 1 ]]; then ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") fi -exit 0; # CONTINUE HERE # TODO iterate over repositories in array REPOSITORIES # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg -if [[ "${REPOSITORY}" == "EESSI" ]]; then - # need to source defaults as late as possible (see other sourcing below) - source ${TOPDIR}/init/eessi_defaults - - # strip "/cvmfs/" from default setting - repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} -else - # TODO implement more flexible specification of repo cfgs - # REPOSITORY => repo-id OR repo-cfg-file (with a single section) OR - # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) - # - # for now, assuming repo-id is defined in config file pointed to - # EESSI_REPOS_CFG_FILE, which is to be copied into the working directory - # (could also become part of the software layer to define multiple - # standard EESSI repositories) - cfg_load ${EESSI_REPOS_CFG_FILE} - - # copy repos.cfg to job directory --> makes it easier to inspect the job - cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. - - # cfg file should include: repo_name, repo_version, config_bundle, - # map { local_filepath -> container_filepath } - # - # repo_name_domain is the domain part of the repo_name, e.g., - # eessi.io for software.eessi.io - # - # where config bundle includes the files (-> target location in container) - # - default.local -> /etc/cvmfs/default.local - # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... - # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf - # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), - # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... - # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} - # a directory that contains the public key to access the repository, key - # itself then doesn't need to be BIND mounted - # - ${repo_name_domain}/${repo_name}.pub - # (-> /etc/cvmfs/keys/${repo_name_domain}/${repo_name}.pub - # the public key to access the repository, key itself is BIND mounted - # via directory ${repo_name_domain} - repo_name=$(cfg_get_value ${REPOSITORY} "repo_name") - # derive domain part from repo_name (everything after first '.') - repo_name_domain=${repo_name#*.} - repo_version=$(cfg_get_value ${REPOSITORY} "repo_version") - config_bundle=$(cfg_get_value ${REPOSITORY} "config_bundle") - config_map=$(cfg_get_value ${REPOSITORY} "config_map") - - # convert config_map into associative array cfg_file_map - cfg_init_file_map "${config_map}" - [[ ${VERBOSE} -eq 1 ]] && cfg_print_map - - # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - # if config_bundle is relative path (no '/' at start) prepend it with - # EESSI_REPOS_CFG_DIR - config_bundle_path= - if [[ ! "${config_bundle}" =~ ^/ ]]; then - config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} - else - config_bundle_path=${config_bundle} - fi +[[ ${VERBOSE} -eq 1 ]] && echo +[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo +for cvmfs_repo in "${REPOSITORIES[@]}" +do + echo "process CVMFS repo spec '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # if cvmfs_repo_name is in cfg_cvmfs_repos, it is a "repository ID" and was + # derived from information in EESSI_REPOS_CFG_FILE, namely the section + # names in that .ini-type file + # in the if-block below, we'll use cfg_repo_id to refer to that ID + # we need to process/provide the config from EESSI_REPOS_CFG_FILE, such + # that the necessary information for accessing a CVMFS repository is made + # available inside the container + if [[ -n "${cfg_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + cfg_repo_id=${cvmfs_repo_name} + + # obtain CVMFS repository name from section for the given ID + cfg_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + # derive domain part from (cfg_)repo_name (everything after first '.') + repo_name_domain=${repo_name#*.} + + # cfg_cvmfs_repos is populated through reading the file pointed to by + # EESSI_REPOS_CFG_FILE. We need to copy that file and data it needs + # into the job's working directory. + + # copy repos.cfg to job directory --> makes it easier to inspect the job + cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + + # cfg file should include sections (one per CVMFS repository to be mounted) + # with each section containing the settings: + # - repo_name, + # - repo_version, + # - config_bundle, and + # - a map { filepath_in_bundle -> container_filepath } + # + # The config_bundle includes the files which are mapped ('->') to a target + # location in container: + # - default.local -> /etc/cvmfs/default.local + # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... + # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf + # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), + # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... + # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} + # a directory that contains the public key to access the repository, key + # itself then doesn't need to be BIND mounted + # - ${repo_name_domain}/${cfg_repo_name}.pub + # (-> /etc/cvmfs/keys/${repo_name_domain}/${cfg_repo_name}.pub + # the public key to access the repository, key itself is BIND mounted + # via directory ${repo_name_domain} + cfg_repo_version=$(cfg_get_value ${cfg_repo_id} "repo_version") + cfg_config_bundle=$(cfg_get_value ${cfg_repo_id} "config_bundle") + cfg_config_map=$(cfg_get_value ${cfg_repo_id} "config_map") + + # convert cfg_config_map into associative array cfg_file_map + cfg_init_file_map "${cfg_config_map}" + [[ ${VERBOSE} -eq 1 ]] && cfg_print_map + + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg and define + # BIND mounts + # check if config_bundle exists, if so, unpack it into + # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${cfg_config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${cfg_config_bundle} + else + config_bundle_path=${cfg_config_bundle} + fi - if [[ ! -r ${config_bundle_path} ]]; then - fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} - fi + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + fi - # only unpack config_bundle if we're not resuming from a previous run - if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg - fi + # only unpack cfg_config_bundle if we're not resuming from a previous run + if [[ -z ${RESUME} ]]; then + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg + fi - for src in "${!cfg_file_map[@]}" - do - target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" - done - export EESSI_VERSION_OVERRIDE=${repo_version} - export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" - # need to source defaults as late as possible (after *_OVERRIDEs) - source ${TOPDIR}/init/eessi_defaults -fi + for src in "${!cfg_file_map[@]}" + do + target=${cfg_file_map[${src}]} + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + done + fi + [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" + [[ ${VERBOSE} -eq 1 ]] && echo +done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to @@ -650,14 +696,16 @@ if [[ ! -z ${http_proxy} ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi +exit 0; # CONTINUE HERE # 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() -# always mount cvmfs-config repo (to get access to software.eessi.io) +# always mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") +# TODO iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) if [[ "${ACCESS}" == "ro" ]]; then export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" From 934059a731f55d5a11cc10b6e3719abf65b1d478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 19 Jun 2024 15:21:27 +0200 Subject: [PATCH 03/11] configure multiple fusemounts --- eessi_container.sh | 123 +++++++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index d561d37792..c549d04d8c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -52,7 +52,6 @@ NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) declare -A cvmfs_repo_settings # CernVM-FS settings -# TODO may need to put them into repository specific map CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" @@ -272,7 +271,7 @@ fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported -# TODO use the value as global setting, suffix to --repository can specify an access mode per repository +# use the value as global setting, suffix to --repository can specify an access mode per repository if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -317,7 +316,7 @@ declare -A listed_repos=() for cvmfs_repo in "${REPOSITORIES[@]}" do cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode - echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" + [[ ${VERBOSE} -eq 1 ]] && echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" # if cvmfs_repo_name is not in eessi_cvmfs_repos, assume it's in cfg_cvmfs_repos # and obtain actual repo_name from config cfg_repo_id='' @@ -409,7 +408,6 @@ if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then fi # 3. set up common vars and directories -# TODO change to be able to support multiple CVMFS repositories # directory structure should be: # ${EESSI_HOST_STORAGE} # |-singularity_cache @@ -528,14 +526,18 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) -BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" +if [[ -z ${SINGULARITY_BIND} ]] ; then + SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" +else + SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi,${SINGULARITY_BIND}" +fi # provide a '/tmp' inside the container -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" +SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then - BIND_PATHS="${BIND_PATHS},${EXTRA_BIND_PATHS}" + SINGULARITY_BIND="${SINGULARITY_BIND},${EXTRA_BIND_PATHS}" fi -[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" declare -a ADDITIONAL_CONTAINER_OPTIONS=() @@ -556,8 +558,8 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then EESSI_USR_LOCAL_CUDA=${EESSI_TMPDIR}/usr-local-cuda mkdir -p ${EESSI_VAR_LOG} mkdir -p ${EESSI_USR_LOCAL_CUDA} - BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" - [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" + SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" + [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" if [[ "${NVIDIA_MODE}" == "install" ]] ; then # No GPU so we need to "trick" Lmod to allow us to load CUDA modules even without a CUDA driver # (this variable means EESSI_OVERRIDE_GPU_CHECK=1 will be set inside the container) @@ -571,16 +573,16 @@ if [[ ${FAKEROOT} -eq 1 ]]; then ADDITIONAL_CONTAINER_OPTIONS+=("--fakeroot") fi -# TODO iterate over repositories in array REPOSITORIES # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg [[ ${VERBOSE} -eq 1 ]] && echo -[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND before processing REPOSITORIES\n SINGULARITY_BIND=${SINGULARITY_BIND}" [[ ${VERBOSE} -eq 1 ]] && echo +# iterate over repositories in array REPOSITORIES for cvmfs_repo in "${REPOSITORIES[@]}" do - echo "process CVMFS repo spec '${cvmfs_repo}'" + [[ ${VERBOSE} -eq 1 ]] && echo "process CVMFS repo spec '${cvmfs_repo}'" # split into name and access mode if ',access=' in $cvmfs_repo if [[ ${cvmfs_repo} == *",access="* ]] ; then cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification @@ -665,17 +667,21 @@ do for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + # if target is alreay BIND mounted, exit with an error + if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then + fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} + fi + export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done fi - [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" + [[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND after processing '${cvmfs_repo}'\n SINGULARITY_BIND=${SINGULARITY_BIND}" [[ ${VERBOSE} -eq 1 ]] && echo done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to -# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND -# MOUNT if it was not yet in BIND_PATHS) +# ${EESSI_TMPDIR}/repos_cfg/default.local on the host (and possibly add a BIND +# MOUNT if it was not yet in SINGULARITY_BIND) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is # http://SOME_HOSTNAME:SOME_PORT/ @@ -691,12 +697,14 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local" [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local - # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then - export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + # if default.local is not BIND mounted into container, add it to SINGULARITY_BIND + src=${EESSI_TMPDIR}/repos_cfg/default.local + target=/etc/cvmfs/default.local + if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then + fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} fi + export SINGULARITY_BIND="${SINGULARITY_BIND},${src}:${target}" fi -exit 0; # CONTINUE HERE # 4. set up vars and dirs specific to a scenario @@ -705,42 +713,59 @@ declare -a EESSI_FUSE_MOUNTS=() # always mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") -# TODO iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) -if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" - - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS -fi +# iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) +for cvmfs_repo in "${REPOSITORIES[@]}" +do + [[ ${VERBOSE} -eq 1 ]] && echo "add fusemount options for CVMFS repo '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # obtain cvmfs_repo_name from EESSI_REPOS_CFG_FILE if cvmfs_repo is in cfg_cvmfs_repos + if [[ ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi -if [[ "${ACCESS}" == "rw" ]]; then - mkdir -p ${EESSI_TMPDIR}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/overlay-work + # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) + if [[ ${cvmfs_repo_access} == "ro" ]] ; then + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" - # set environment variables for fuse mounts in Singularity container - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + elif [[ ${cvmfs_repo_access} == "rw" ]] ; then + # use repo-specific overlay directories + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + # set environment variables for fuse mounts in Singularity container + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" - EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" - EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") - export EESSI_FUSE_MOUNTS -fi + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + export EESSI_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") + export EESSI_FUSE_MOUNTS + else + echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" + exit ${REPOSITORY_ERROR_EXITCODE} + fi +done # 5. run container -# final settings -if [[ -z ${SINGULARITY_BIND} ]]; then - export SINGULARITY_BIND="${BIND_PATHS}" -else - export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" -fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" # pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) From 32f413328cd3b8f1da7ea7521f7458c20cd7232b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 19 Jun 2024 16:02:40 +0200 Subject: [PATCH 04/11] revert back changes related to BIND_PATHS and SINGULARITY_BIND + some polishing --- eessi_container.sh | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index c549d04d8c..163792f622 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -416,12 +416,10 @@ fi # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} # |-CVMFS_REPO_1 -# | |-repo_settings (name, access_mode, host_injections) # | |-overlay-upper # | |-overlay-work # | |-opt-eessi (unless otherwise specificed for host_injections) # |-CVMFS_REPO_n -# |-repo_settings (name, access_mode, host_injections) # |-overlay-upper # |-overlay-work # |-opt-eessi (unless otherwise specificed for host_injections) @@ -526,18 +524,15 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) -if [[ -z ${SINGULARITY_BIND} ]] ; then - SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" -else - SINGULARITY_BIND="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi,${SINGULARITY_BIND}" -fi +BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" + # provide a '/tmp' inside the container -SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then - SINGULARITY_BIND="${SINGULARITY_BIND},${EXTRA_BIND_PATHS}" + BIND_PATHS="${BIND_PATHS},${EXTRA_BIND_PATHS}" fi -[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" declare -a ADDITIONAL_CONTAINER_OPTIONS=() @@ -558,8 +553,8 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then EESSI_USR_LOCAL_CUDA=${EESSI_TMPDIR}/usr-local-cuda mkdir -p ${EESSI_VAR_LOG} mkdir -p ${EESSI_USR_LOCAL_CUDA} - SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" - [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" + BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" + [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" if [[ "${NVIDIA_MODE}" == "install" ]] ; then # No GPU so we need to "trick" Lmod to allow us to load CUDA modules even without a CUDA driver # (this variable means EESSI_OVERRIDE_GPU_CHECK=1 will be set inside the container) @@ -577,7 +572,7 @@ fi # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg [[ ${VERBOSE} -eq 1 ]] && echo -[[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND before processing REPOSITORIES\n SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" [[ ${VERBOSE} -eq 1 ]] && echo # iterate over repositories in array REPOSITORIES for cvmfs_repo in "${REPOSITORIES[@]}" @@ -668,20 +663,20 @@ do do target=${cfg_file_map[${src}]} # if target is alreay BIND mounted, exit with an error - if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then - fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} fi - export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done fi - [[ ${VERBOSE} -eq 1 ]] && echo -e "SINGULARITY_BIND after processing '${cvmfs_repo}'\n SINGULARITY_BIND=${SINGULARITY_BIND}" + [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" [[ ${VERBOSE} -eq 1 ]] && echo done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to # ${EESSI_TMPDIR}/repos_cfg/default.local on the host (and possibly add a BIND -# MOUNT if it was not yet in SINGULARITY_BIND) +# MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is # http://SOME_HOSTNAME:SOME_PORT/ @@ -697,13 +692,13 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local" [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local - # if default.local is not BIND mounted into container, add it to SINGULARITY_BIND + # if default.local is not BIND mounted into container, add it to BIND_PATHS src=${EESSI_TMPDIR}/repos_cfg/default.local target=/etc/cvmfs/default.local - if [[ ${SINGULARITY_BIND} =~ "${target}" ]]; then - fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${SINGULARITY_BIND}')" ${REPOSITORY_ERROR_EXITCODE} + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} fi - export SINGULARITY_BIND="${SINGULARITY_BIND},${src}:${target}" + BIND_PATHS="${BIND_PATHS},${src}:${target}" fi # 4. set up vars and dirs specific to a scenario @@ -744,6 +739,7 @@ do # use repo-specific overlay directories mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" @@ -766,6 +762,12 @@ do done # 5. run container +# final settings +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BIND_PATHS}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" +fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" # pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) From 07de7008e60586d146cd4986de9d0a4f7e505731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Thu, 20 Jun 2024 09:07:24 +0200 Subject: [PATCH 05/11] store repository settings under tmp --- eessi_container.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 163792f622..eaa4f59af9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -416,10 +416,12 @@ fi # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} # |-CVMFS_REPO_1 +# | |-repo_settings.sh (name, id, access, host_injections) # | |-overlay-upper # | |-overlay-work # | |-opt-eessi (unless otherwise specificed for host_injections) # |-CVMFS_REPO_n +# |-repo_settings.sh (name, id, access, host_injections) # |-overlay-upper # |-overlay-work # |-opt-eessi (unless otherwise specificed for host_injections) @@ -711,6 +713,7 @@ EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/ # iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) for cvmfs_repo in "${REPOSITORIES[@]}" do + unset cfg_repo_id [[ ${VERBOSE} -eq 1 ]] && echo "add fusemount options for CVMFS repo '${cvmfs_repo}'" # split into name and access mode if ',access=' in $cvmfs_repo if [[ ${cvmfs_repo} == *",access="* ]] ; then @@ -729,6 +732,9 @@ do cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi + # always create a directory for the repository (e.g., to store settings, ...) + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} + # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) if [[ ${cvmfs_repo_access} == "ro" ]] ; then export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" @@ -759,6 +765,20 @@ do echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" exit ${REPOSITORY_ERROR_EXITCODE} fi + # create repo_settings.sh file in ${EESSI_TMPDIR}/${cvmfs_repo_name} to store + # (intention is that the file could be just sourced to obtain the settings) + # repo_name = ${cvmfs_repo_name} + # repo_id = ${cfg_repo_id} # empty if not an EESSI repo + # repo_access = ${cvmfs_repo_access} + # repo_host_injections = [ {"src_path":"target_path"}... ] # TODO + settings= + #[[ -n ${cfg_repo_id} ]] && settings="[${cvmfs_repo_name}]\n" || settings="[${cfg_repo_id}]\n" + settings="${settings}repo_name = ${cvmfs_repo_name}\n" + settings="${settings}repo_id = ${cfg_repo_id}\n" + settings="${settings}repo_access = ${cvmfs_repo_access}\n" + # TODO iterate over host_injections (first need means to define them (globally and/or per repository) + # settings="${settings}repo_host_injections = ${host_injections}\n" + echo -e "${settings}" > ${EESSI_TMPDIR}/${cvmfs_repo_name}/repo_settings.sh done # 5. run container From d9c1fb1104f19e741b8ac2a5a652eb22e07bf241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Tue, 25 Jun 2024 10:17:51 +0200 Subject: [PATCH 06/11] removed unnecessary variable --- eessi_container.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index eaa4f59af9..4b7b4eb719 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -48,9 +48,6 @@ HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12)) -# we use an associative array for storing sets of settings per CVMFS repository -declare -A cvmfs_repo_settings - # CernVM-FS settings CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" From e25cd1e720a3a0e787409093d561d5892929d10a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Tue, 25 Jun 2024 10:23:18 +0200 Subject: [PATCH 07/11] add more information about repository-specific access modes --- eessi_container.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 4b7b4eb719..9d2ce6c25f 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,7 +70,8 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" display_help() { echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -a | --access {ro,rw} - sets access globally for all CVMFS repositories:" + echo " ro (read-only), rw (read & write) [default: ro]" echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." echo " To specify multiple bind paths, separate by comma." echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" @@ -89,7 +90,9 @@ display_help() { echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use; can be given multiple times" + echo " repository to use; can be given multiple times;" + echo " CFG may include a suffix ';access={ro,rw}' to" + echo " overwrite the global access mode for this repository" echo " [default: software.eessi.io via CVMFS config available" echo " via default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," From c895bf16668c588d7eec827092849d883ce10bf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 13:24:12 +0200 Subject: [PATCH 08/11] remove TODO for using separate cache dirs for each repo --- eessi_container.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9d2ce6c25f..94de82df0f 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -510,7 +510,6 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" # set env vars and create directories for CernVM-FS -# TODO need to use separate values for separate repos? EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} From 417f4dd9b82795520e5923a9c1532ed04f384747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 13:24:44 +0200 Subject: [PATCH 09/11] fix usage line for access permissions --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 94de82df0f..e404b7ee18 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -91,7 +91,7 @@ display_help() { echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use; can be given multiple times;" - echo " CFG may include a suffix ';access={ro,rw}' to" + echo " CFG may include a suffix ',access={ro,rw}' to" echo " overwrite the global access mode for this repository" echo " [default: software.eessi.io via CVMFS config available" echo " via default container, see --container]" From d8cf6a60540b36b5c3e64c687af5d1482df021a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 13:41:40 +0200 Subject: [PATCH 10/11] check for software.eessi.io in the default --list-repos output --- .github/workflows/test_eessi_container_script.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index d31feb23fe..c0346774cb 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -45,7 +45,8 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then outfile=out_listrepos.txt ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} # test use of --list-repos with custom repos.cfg elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then From aa9895e44ac1afd5a44524d004161f3804449a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 4 Jul 2024 15:27:27 +0200 Subject: [PATCH 11/11] fix regex to find the custom EESSI version in the output of --list-repos --- .github/workflows/test_eessi_container_script.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index c0346774cb..d44788927f 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -58,11 +58,12 @@ jobs: echo "[EESSI/20HT.TP]" >> cfg/repos.cfg echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile2} - grep "[EESSI/2023.02]" ${outfile2} + grep "EESSI/20AB.CD" ${outfile2} # test use of --mode run elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then