From 679fe863a9c42041aba19fbf66923ecc73db3dca Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 30 Dec 2022 05:19:36 +0100 Subject: [PATCH 01/75] use single source file for pilot version and repository - adds a new configuration file init/eessi_defaults which sets EESSI_PILOT_VERSION to a default or EESSI_PILOT_VERSION_OVERRIDE and EESSI_CVMFS_REPO to a default or EESSI_CVMFS_REPO_OVERRIDE - existing init files and scripts that used hardcoded values are adjusted to source the new configuration file - a comment in EESSI-pilot-install-software.sh is updated that not a hardcoded version but the one from init/eessi_defaults is used - a simple test is added which checks if the default values in init/eessi_defaults correspond to values used in README.md --- .github/workflows/tests_readme.yml | 31 ++++++++++++++++++++++++++++++ EESSI-pilot-install-software.sh | 2 +- build_container.sh | 10 ++++++++-- create_tarball.sh | 5 ++++- init/eessi_defaults | 13 +++++++++++++ init/minimal_eessi_env | 10 ++++++++-- install_software_layer.sh | 3 ++- run_in_compat_layer_env.sh | 7 ++++++- 8 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/tests_readme.yml create mode 100644 init/eessi_defaults diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml new file mode 100644 index 0000000000..29dedcc3ff --- /dev/null +++ b/.github/workflows/tests_readme.yml @@ -0,0 +1,31 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for consistency of README.md +on: + push: + paths: + - README.md + - init/eessi_defaults + + pull_request: + branches: + - main + paths: + - README.md + - init/eessi_defaults +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + + - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults + run: | + source init/eessi_defaults + grep "${EESSI_PILOT_VERSION}" README.md + + - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults + run: | + source init/eessi_defaults + grep "${EESSI_CVMFS_REPO}" README.md diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 10311b7a15..d4d4fd28e8 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to install EESSI pilot software stack (version 2021.12) +# Script to install EESSI pilot software stack (version set through init/eessi_defaults) # see example parsing of command line arguments at # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop diff --git a/build_container.sh b/build_container.sh index d7e7256b4e..e087b9f7be 100755 --- a/build_container.sh +++ b/build_container.sh @@ -1,5 +1,7 @@ #!/bin/bash +base_dir=$(dirname $(realpath $0)) + BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10" if [ $# -lt 2 ]; then @@ -39,9 +41,13 @@ if [ -z $SINGULARITY_HOME ]; then export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER" fi +source ${base_dir}/init/eessi_defaults +# strip "/cvmfs/" from default setting +repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} + # set environment variables for fuse mounts in Singularity container -export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org" -export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org" +export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" +export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/${repo_name} -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work ${EESSI_CVMFS_REPO}" if [ "$SHELL_OR_RUN" == "shell" ]; then # start shell in Singularity container, with EESSI repository mounted with writable overlay diff --git a/create_tarball.sh b/create_tarball.sh index 619d6ea2e1..142df60a94 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -2,6 +2,8 @@ set -e +base_dir=$(dirname $(realpath $0)) + if [ $# -ne 4 ]; then echo "ERROR: Usage: $0 " >&2 exit 1 @@ -15,7 +17,8 @@ tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" os="linux" -cvmfs_repo="/cvmfs/pilot.eessi-hpc.org" +source ${base_dir}/init/eessi_defaults +cvmfs_repo=${EESSI_CVMFS_REPO} software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then diff --git a/init/eessi_defaults b/init/eessi_defaults new file mode 100644 index 0000000000..1b5ce07fb1 --- /dev/null +++ b/init/eessi_defaults @@ -0,0 +1,13 @@ +# define default values for some EESSI_* environment variables +# +# This file is part of the EESSI software layer, +# see https://github.com/EESSI/software-layer +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" +export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" + diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 39478d5c4c..b7cb7c5e9e 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -1,7 +1,13 @@ # define minimal EESSI environment, without relying on external scripts +# +# this script is *sourced*, not executed, so can't rely on $0 to determine path to self +# $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 +EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) + +# set up defaults: EESSI_CVMFS_REPO, EESSI_PILOT_VERSION +# script takes *_OVERRIDEs into account +source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_CVMFS_REPO="/cvmfs/pilot.eessi-hpc.org" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_PILOT_VERSION if [[ $(uname -s) == 'Linux' ]]; then diff --git a/install_software_layer.sh b/install_software_layer.sh index e0b2f785d5..bf3006a4a0 100755 --- a/install_software_layer.sh +++ b/install_software_layer.sh @@ -1,3 +1,4 @@ #!/bin/bash -export EESSI_PILOT_VERSION='2021.12' +base_dir=$(dirname $(realpath $0)) +source ${base_dir}/init/eessi_defaults ./run_in_compat_layer_env.sh ./EESSI-pilot-install-software.sh "$@" diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 118aac65d5..41a7be1ca0 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -1,9 +1,14 @@ #!/bin/bash + +base_dir=$(dirname $(realpath $0)) +source ${base_dir}/init/eessi_defaults + +BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10" if [ -z $EESSI_PILOT_VERSION ]; then echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 exit 1 fi -EESSI_COMPAT_LAYER_DIR="/cvmfs/pilot.eessi-hpc.org/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 exit 1 From 2caad9bc9cc99544542725894c7ee11c8b6ea701 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 20:42:40 +0100 Subject: [PATCH 02/75] initial version of unified script - should support existing ways to access EESSI - no additional features implemented yet (--help may hint at new features) --- eessi_container.sh | 311 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100755 eessi_container.sh diff --git a/eessi_container.sh b/eessi_container.sh new file mode 100755 index 0000000000..9219581582 --- /dev/null +++ b/eessi_container.sh @@ -0,0 +1,311 @@ +#!/bin/bash +# +# unified script to access EESSI in different scenarios: read-only +# for just using EESSI, read & write for building software to be +# added to the software stack +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# script overview +# -. initial settings & exit codes +# 0. parse args +# 1. check if argument values are valid +# 2. set up local disk/tmp +# 3. set up common vars and directories +# 4. set up vars specific to a scenario +# 5. initialize local disk/tmp from previous run if provided +# 6. run container + +# -. initial settings & exit codes +base_dir=$(dirname $(realpath $0)) + +# functions +function echo_red() { + echo -e "\e[31m$1\e[0m" +} + +function fatal_error() { + echo_red "ERROR: ${1}" >&2 + exit ${2} +} + +# exit codes: bitwise shift codes to allow for combination of exit codes +ANY_ERROR_EXITCODE=1 +CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1)) +ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) +CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) +LOCAL_DISK_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) +MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) +PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) +REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) + +# CernVM-FS settings +CVMFS_VAR_LIB="var-lib-cvmfs" +CVMFS_VAR_RUN="var-run-cvmfs" + + +# 0. parse args +# see example parsing of command line arguments at +# https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash + +display_help() { + echo "usage: $0 [OPTIONS] [SCRIPT]" + echo " OPTIONS:" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMAGE - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian10]" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" + echo " -h | --help - display this usage information [default: false]" + echo " -i | --info - display configured repositories [default: false]" + echo " -l | --local-disk DIR - directory space on local machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode {shell,run} - shell (launch interactive shell)" + echo " run (run a script) [default: shell]" + echo " -p | --previous-run PRUN - init local disk with data from previous run" + echo " format is PATH[:TAR/ZIP] where PATH is pointing" + echo " to the previously used local disk, and TAR/ZIP" + echo " is used to initialize the local disk if PATH" + echo " doesn't exist currently; if PATH exists and" + echo " a TAR/ZIP is provided an error is reported" + echo " [default: not set]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]" + echo + echo " If value for --mode is 'run', the SCRIPT provided is executed." +} + +# set defaults for command line arguments +ACCESS="ro" +CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +DRY_RUN=0 +INFO=0 +LOCAL_DISK= +MODE="shell" +PREVIOUS_RUN= +REPOSITORY="EESSI-pilot" +HTTP_PROXY= +HTTPS_PROXY= +RUN_SCRIPT_AND_ARGS= + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -a|--access) + ACCESS="$2" + shift 2 + ;; + -c|--container) + CONTAINER="$2" + shift 2 + ;; + -d|--dry-run) + DRY_RUN=1 + ;; + -h|--help) + display_help + exit 0 + ;; + -i|--info) + INFO=1 + ;; + -l|--local-disk) + LOCAL_DISK="$2" + #EESSI_TMPDIR="$2" + shift 2 + ;; + -m|--mode) + MODE="$2" + shift 2 + ;; + -p|--previous-run) + PREVIOUS_RUN="$2" + shift 2 + ;; + -x|--http-proxy) + HTTP_PROXY="$2" + export http_proxy=${HTTP_PROXY} + shift 2 + ;; + -y|--https-proxy) + HTTPS_PROXY="$2" + export https_proxy=${HTTPS_PROXY} + shift 2 + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + + +# 1. check if argument values are valid +# (arg -a|--access) check if ACCESS is supported +if [[ ${ACCESS} != "ro" -a ${ACCESS} != "rw" ]]; then + fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" +fi + +# TODO (arg -c|--container) check container (is it a file or URL & access those) +# CONTAINER_ERROR_EXITCODE + +# TODO (arg -l|--local-disk) check if it exists, if user has write permission, +# if it contains no data, etc. +# LOCAL_DISK_ERROR_EXITCODE + +# (arg -m|--mode) check if MODE is known +if [[ ${MODE} != "shell" -a ${MODE} != "run" ]]; then + fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" +fi + +# TODO (arg -p|--previous-run) check if it exists, if user has read permission, +# if it contains data from a previous run +# PREVIOUS_RUN_ERROR_EXITCODE + +# TODO (arg -r|--repository) check if repository is known +# REPOSITORY_UNKNOWN_EXITCODE + +# TODO (arg -x|--http-proxy) check if http proxy is accessible +# HTTP_PROXY_ERROR_EXITCODE + +# TODO (arg -y|--https-proxy) check if https proxy is accessible +# HTTPS_PROXY_ERROR_EXITCODE + +# check if a script is provided if mode is 'run' +if [[ "${MODE}" == "run" ]]; then + if [[ $# -eq 0 ]]; then + fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}" + else + RUN_SCRIPT_AND_ARGS=$@ + fi +fi + + +# 2. set up local disk/tmp +# as location for temporary data use in the following order +# a. command line argument -l|--local_disk +# b. env var TMPDIR +# c. /tmp +# note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK +# if LOCAL_DISK is not empty +# note, (b) & (c) are automatically ensured by using mktemp -d to create +# a temporary directory +# note, if previous run is used the name of the temporary directory +# should be identical to previous run, ie, then we don't create a new +# temporary directory +if [[ ! -z ${LOCAL_DISK} ]]; then + TMPDIR=${LOCAL_DISK} +fi +if [[ ! -z ${TMPDIR} ]]; then + # TODO check if TMPDIR already exists + EESSI_LOCAL_DISK=${TMPDIR} +fi +if [[ -z ${TMPDIR} ]]; then + # mktemp falls back to using /tmp if TMPDIR is empty + # TODO check if /tmp is writable, large enough and usable (different + # features for ro-access and rw-access) + echo "skipping sanity checks for /tmp" +fi +EESSI_LOCAL_DISK=$(mktemp -d eessi.XXXXXXXXXX) +echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." + + +# 3. set up common vars and directories +# directory structure should be: +# ${EESSI_LOCAL_DISK} +# |-singularity_cache +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} + +source ${base_dir}/init/eessi_defaults + +# configure Singularity +export SINGULARITY_CACHEDIR=${EESSI_LOCAL_DISK}/singularity_cache +mkdir -p ${SINGULARITY_CACHEDIR} +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" + +# set env vars and create directories for CernVM-FS +EESSI_CVMFS_VAR_LIB=${EESSI_LOCAL_DISK}/${CVMFS_VAR_LIB} +EESSI_CVMFS_VAR_RUN=${EESSI_LOCAL_DISK}/${CVMFS_VAR_RUN} +mkdir -p ${EESSI_CVMFS_VAR_LIB} +mkdir -p ${EESSI_CVMFS_VAR_RUN} +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" + +# tmp dir for EESSI +EESSI_TMPDIR=${EESSI_LOCAL_DISK}/tmp +mkdir -p ${EESSI_TMPDIR} +[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" + +# allow that SINGULARITY_HOME is defined before script is run +if [[ -z ${SINGULARITY_HOME} ]]; then + export SINGULARITY_HOME="${EESSI_LOCAL_DISK}/home:/home/${USER}" + mkdir -p ${EESSI_LOCAL_DISK}/home + [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" +fi + +# define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) +BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvfms,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" + + +# 4. set up vars and dirs specific to a scenario + +# strip "/cvmfs/" from default setting +repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} + +if [[ "${ACCESS}" == "ro" ]]; then + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}" + export EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" +fi + +if [[ "${ACCESS}" == "rw" ]]; then + EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper + EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work + mkdir -p ${EESSI_CMVFS_OVERLAY_UPPER} + mkdir -p ${EESSI_CMVFS_OVERLAY_WORK} + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" + + # set environment variables for fuse mounts in Singularity container + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}" + EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_PILOT_WRITABLE_OVERLAY + + EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" + EESSI_FUSE_MOUNTS+=" --fusemount ${EESSI_PILOT_WRITABLE_OVERLAY}" + export EESSI_FUSE_MOUNTS +fi + + +# 5. initialize local disk/tmp from previous run if provided + + +# 6. run container +echo "Launching container with command (next line):" +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} From 6f904cee6e6aa9d83e79e0bf7eabbc329885561b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 20:54:01 +0100 Subject: [PATCH 03/75] fix test syntax --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9219581582..273d3bb297 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -162,7 +162,7 @@ set -- "${POSITIONAL_ARGS[@]}" # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported -if [[ ${ACCESS} != "ro" -a ${ACCESS} != "rw" ]]; then +if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -174,7 +174,7 @@ fi # LOCAL_DISK_ERROR_EXITCODE # (arg -m|--mode) check if MODE is known -if [[ ${MODE} != "shell" -a ${MODE} != "run" ]]; then +if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi From d6aae8d03247f2fe0e571989070ff94764dfdcd6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 21:44:53 +0100 Subject: [PATCH 04/75] use array for EESSI_FUSE_MOUNTS, fix var name typo --- eessi_container.sh | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 273d3bb297..23d6cd4bf5 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -274,31 +274,30 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" # strip "/cvmfs/" from default setting repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} +declare -a EESSI_FUSE_MOUNTS=() if [[ "${ACCESS}" == "ro" ]]; then export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}" - export EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") fi if [[ "${ACCESS}" == "rw" ]]; then EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work - mkdir -p ${EESSI_CMVFS_OVERLAY_UPPER} - mkdir -p ${EESSI_CMVFS_OVERLAY_WORK} + mkdir -p ${EESSI_CVMFS_OVERLAY_UPPER} + mkdir -p ${EESSI_CVMFS_OVERLAY_WORK} [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" # set environment variables for fuse mounts in Singularity container - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}" EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_PILOT_WRITABLE_OVERLAY - - EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" - EESSI_FUSE_MOUNTS+=" --fusemount ${EESSI_PILOT_WRITABLE_OVERLAY}" - export EESSI_FUSE_MOUNTS + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") fi @@ -307,5 +306,5 @@ fi # 6. run container echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" -singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} From 4194421b5075bfc8714a5d6d0f3cd554d35992aa Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 21:46:01 +0100 Subject: [PATCH 05/75] add missing shift operations --- eessi_container.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 23d6cd4bf5..fcc5cc1e31 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -116,6 +116,7 @@ while [[ $# -gt 0 ]]; do ;; -d|--dry-run) DRY_RUN=1 + shift 1 ;; -h|--help) display_help @@ -123,6 +124,7 @@ while [[ $# -gt 0 ]]; do ;; -i|--info) INFO=1 + shift 1 ;; -l|--local-disk) LOCAL_DISK="$2" From 8b2f7301fc9cf7e2b583397a4f603ab74f4efcc0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 22:10:09 +0100 Subject: [PATCH 06/75] fixing creation of temporary directory --- eessi_container.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index fcc5cc1e31..3d64585d0d 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -210,17 +210,20 @@ fi # c. /tmp # note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK # if LOCAL_DISK is not empty -# note, (b) & (c) are automatically ensured by using mktemp -d to create -# a temporary directory +# note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to +# create a temporary directory # note, if previous run is used the name of the temporary directory # should be identical to previous run, ie, then we don't create a new # temporary directory if [[ ! -z ${LOCAL_DISK} ]]; then TMPDIR=${LOCAL_DISK} + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} fi if [[ ! -z ${TMPDIR} ]]; then # TODO check if TMPDIR already exists - EESSI_LOCAL_DISK=${TMPDIR} + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} fi if [[ -z ${TMPDIR} ]]; then # mktemp falls back to using /tmp if TMPDIR is empty @@ -228,7 +231,7 @@ if [[ -z ${TMPDIR} ]]; then # features for ro-access and rw-access) echo "skipping sanity checks for /tmp" fi -EESSI_LOCAL_DISK=$(mktemp -d eessi.XXXXXXXXXX) +EESSI_LOCAL_DISK=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." From 6aa5cac8d7335e47120c5504de2fb8e1aa86f95e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 22:32:10 +0100 Subject: [PATCH 07/75] fix BIND typo, define SINGULARITY_BIND --- eessi_container.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 3d64585d0d..0425d8759b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -270,8 +270,9 @@ if [[ -z ${SINGULARITY_HOME} ]]; then fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) -BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvfms,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" +BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" +[[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # 4. set up vars and dirs specific to a scenario @@ -310,6 +311,14 @@ fi # 6. run container +# final settings +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BIND_PATHS}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" +fi +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" + echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} From 3a2bf51687bb1c9b8e39518e2481939e84dfc751 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 23:48:28 +0100 Subject: [PATCH 08/75] fix tmp dir for overlay fs --- eessi_container.sh | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 0425d8759b..1be3b566db 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -241,31 +241,34 @@ echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." # |-singularity_cache # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} +# |-overlay-upper +# |-overlay-work +# |-home source ${base_dir}/init/eessi_defaults +# tmp dir for EESSI +EESSI_TMPDIR=${EESSI_LOCAL_DISK} +mkdir -p ${EESSI_TMPDIR} +[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" + # configure Singularity -export SINGULARITY_CACHEDIR=${EESSI_LOCAL_DISK}/singularity_cache +export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache mkdir -p ${SINGULARITY_CACHEDIR} [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" # set env vars and create directories for CernVM-FS -EESSI_CVMFS_VAR_LIB=${EESSI_LOCAL_DISK}/${CVMFS_VAR_LIB} -EESSI_CVMFS_VAR_RUN=${EESSI_LOCAL_DISK}/${CVMFS_VAR_RUN} +EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} +EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} mkdir -p ${EESSI_CVMFS_VAR_RUN} [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" -# tmp dir for EESSI -EESSI_TMPDIR=${EESSI_LOCAL_DISK}/tmp -mkdir -p ${EESSI_TMPDIR} -[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" - # allow that SINGULARITY_HOME is defined before script is run if [[ -z ${SINGULARITY_HOME} ]]; then - export SINGULARITY_HOME="${EESSI_LOCAL_DISK}/home:/home/${USER}" - mkdir -p ${EESSI_LOCAL_DISK}/home + export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" + mkdir -p ${EESSI_TMPDIR}/home [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" fi @@ -287,10 +290,10 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper - EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work - mkdir -p ${EESSI_CVMFS_OVERLAY_UPPER} - mkdir -p ${EESSI_CVMFS_OVERLAY_WORK} + EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper + EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work + mkdir -p ${EESSI_TMPDIR}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/overlay-work [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" @@ -300,8 +303,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") fi From 3fbea5e4faa3f4acbe31108fe8f08e2a5f088405 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 2 Jan 2023 18:53:53 +0100 Subject: [PATCH 09/75] fixing export bug, improving readability - added missing export for TMPDIR - moved functions to utils.sh - renamed local-disk/LOCAL_DISK to host-storage/HOST_STORAGE --- eessi_container.sh | 58 ++++++++++++++++++----------------------- utils.sh | 65 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1be3b566db..b597b17647 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -16,37 +16,29 @@ # -. initial settings & exit codes # 0. parse args # 1. check if argument values are valid -# 2. set up local disk/tmp +# 2. set up host storage/tmp # 3. set up common vars and directories # 4. set up vars specific to a scenario -# 5. initialize local disk/tmp from previous run if provided +# 5. initialize host storage/tmp from previous run if provided # 6. run container # -. initial settings & exit codes base_dir=$(dirname $(realpath $0)) -# functions -function echo_red() { - echo -e "\e[31m$1\e[0m" -} - -function fatal_error() { - echo_red "ERROR: ${1}" >&2 - exit ${2} -} +source ${base_dir}/utils.sh # exit codes: bitwise shift codes to allow for combination of exit codes -ANY_ERROR_EXITCODE=1 +# ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1)) ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) -LOCAL_DISK_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) +HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) -HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings CVMFS_VAR_LIB="var-lib-cvmfs" @@ -68,7 +60,7 @@ display_help() { echo " print information about setup [default: false]" echo " -h | --help - display this usage information [default: false]" echo " -i | --info - display configured repositories [default: false]" - echo " -l | --local-disk DIR - directory space on local machine (used for" + echo " -l | --host-storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -m | --mode {shell,run} - shell (launch interactive shell)" echo " run (run a script) [default: shell]" @@ -94,7 +86,7 @@ ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian10" DRY_RUN=0 INFO=0 -LOCAL_DISK= +HOST_STORAGE= MODE="shell" PREVIOUS_RUN= REPOSITORY="EESSI-pilot" @@ -126,8 +118,8 @@ while [[ $# -gt 0 ]]; do INFO=1 shift 1 ;; - -l|--local-disk) - LOCAL_DISK="$2" + -l|--host-storage) + HOST_STORAGE="$2" #EESSI_TMPDIR="$2" shift 2 ;; @@ -171,9 +163,9 @@ fi # TODO (arg -c|--container) check container (is it a file or URL & access those) # CONTAINER_ERROR_EXITCODE -# TODO (arg -l|--local-disk) check if it exists, if user has write permission, +# TODO (arg -l|--host-storage) check if it exists, if user has write permission, # if it contains no data, etc. -# LOCAL_DISK_ERROR_EXITCODE +# HOST_STORAGE_ERROR_EXITCODE # (arg -m|--mode) check if MODE is known if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then @@ -203,20 +195,20 @@ if [[ "${MODE}" == "run" ]]; then fi -# 2. set up local disk/tmp +# 2. set up host storage/tmp # as location for temporary data use in the following order -# a. command line argument -l|--local_disk +# a. command line argument -l|--host-storage # b. env var TMPDIR # c. /tmp -# note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK -# if LOCAL_DISK is not empty +# note, we ensure that (a) takes precedence by setting TMPDIR to HOST_STORAGE +# if HOST_STORAGE is not empty # note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to # create a temporary directory # note, if previous run is used the name of the temporary directory # should be identical to previous run, ie, then we don't create a new # temporary directory -if [[ ! -z ${LOCAL_DISK} ]]; then - TMPDIR=${LOCAL_DISK} +if [[ ! -z ${HOST_STORAGE} ]]; then + export TMPDIR=${HOST_STORAGE} # mktemp fails if TMPDIR does not exist, so let's create it mkdir -p ${TMPDIR} fi @@ -231,13 +223,13 @@ if [[ -z ${TMPDIR} ]]; then # features for ro-access and rw-access) echo "skipping sanity checks for /tmp" fi -EESSI_LOCAL_DISK=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) -echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." +EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) +echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." # 3. set up common vars and directories # directory structure should be: -# ${EESSI_LOCAL_DISK} +# ${EESSI_HOST_STORAGE} # |-singularity_cache # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} @@ -248,7 +240,7 @@ echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." source ${base_dir}/init/eessi_defaults # tmp dir for EESSI -EESSI_TMPDIR=${EESSI_LOCAL_DISK} +EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" @@ -310,7 +302,7 @@ if [[ "${ACCESS}" == "rw" ]]; then fi -# 5. initialize local disk/tmp from previous run if provided +# 5. initialize host storage/tmp from previous run if provided # 6. run container diff --git a/utils.sh b/utils.sh index 0c98c86ec4..82dfd4a345 100644 --- a/utils.sh +++ b/utils.sh @@ -10,9 +10,14 @@ function echo_yellow() { echo -e "\e[33m$1\e[0m" } +ANY_ERROR_EXITCODE=1 function fatal_error() { echo_red "ERROR: $1" >&2 - exit 1 + if [[ $# -gt 1 ]]; then + exit $2 + else + exit "${ANY_ERROR_EXITCODE}" + fi } function check_exit_code { @@ -26,3 +31,61 @@ function check_exit_code { fatal_error "${fail_msg}" fi } + +declare -A repo_cfg + +function cfg_get_section { + if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then + echo ${BASH_REMATCH[2]} + else + echo "" + fi +} + +function cfg_get_key_value { + if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then + echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" + else + echo "" + fi +} + +function cfg_load { + local cur_section="" + local cur_key="" + local cur_val="" + IFS= + while read -r line; do + new_section=$(cfg_get_section $line) + # got a new section + if [[ -n "$new_section" ]]; then + cur_section=$new_section + # not a section, try a key value + else + val=$(cfg_get_key_value $line) + # trim leading and trailing spaces as well + cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + if [[ -n "$cur_key" ]]; then + # section + key is the associative in bash array, the field separator is space + repo_cfg[${cur_section} ${cur_key}]=$cur_val + fi + fi + done <$1 +} + +function cfg_print { + for i in "${!repo_cfg[@]}" + do + # split the associative key in to section and key + echo -n "section : $(echo $i | cut -f1 -d ' ');" + echo -n "key : $(echo $i | cut -f2 -d ' ');" + echo "value: ${repo_cfg[$i]}" + done +} + +function cfg_get_value { + section=$1 + key=$2 + echo "${repo_cfg[$section $key]}" +} From 84daaa36c1e06baf3cd0de9f93ac9cf26c354cd3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 14:43:15 +0100 Subject: [PATCH 10/75] moving cfg handling to separate file cfg_files.sh --- cfg_files.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ utils.sh | 58 ----------------------------------------- 2 files changed, 73 insertions(+), 58 deletions(-) create mode 100644 cfg_files.sh diff --git a/cfg_files.sh b/cfg_files.sh new file mode 100644 index 0000000000..c2c549d0bf --- /dev/null +++ b/cfg_files.sh @@ -0,0 +1,73 @@ +# functions for working with ini/cfg files +# +# This file is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + + +# global variables +# -a -> indexed array +# -A -> associative array +declare -A cfg_repos + + +# functions +function cfg_get_section { + if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then + echo ${BASH_REMATCH[2]} + else + echo "" + fi +} + +function cfg_get_key_value { + if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then + echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" + else + echo "" + fi +} + +function cfg_load { + local cur_section="" + local cur_key="" + local cur_val="" + IFS= + while read -r line; do + new_section=$(cfg_get_section $line) + # got a new section + if [[ -n "$new_section" ]]; then + cur_section=$new_section + # not a section, try a key value + else + val=$(cfg_get_key_value $line) + # trim leading and trailing spaces as well + cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + if [[ -n "$cur_key" ]]; then + # section + key is the associative in bash array, the field separator is space + repo_cfg[${cur_section} ${cur_key}]=$cur_val + fi + fi + done <$1 +} + +function cfg_print { + for i in "${!repo_cfg[@]}" + do + # split the associative key in to section and key + echo -n "section : $(echo $i | cut -f1 -d ' ');" + echo -n "key : $(echo $i | cut -f2 -d ' ');" + echo "value: ${repo_cfg[$i]}" + done +} + +function cfg_get_value { + section=$1 + key=$2 + echo "${repo_cfg[$section $key]}" +} diff --git a/utils.sh b/utils.sh index 82dfd4a345..5d8455bb68 100644 --- a/utils.sh +++ b/utils.sh @@ -31,61 +31,3 @@ function check_exit_code { fatal_error "${fail_msg}" fi } - -declare -A repo_cfg - -function cfg_get_section { - if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then - echo ${BASH_REMATCH[2]} - else - echo "" - fi -} - -function cfg_get_key_value { - if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then - echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" - else - echo "" - fi -} - -function cfg_load { - local cur_section="" - local cur_key="" - local cur_val="" - IFS= - while read -r line; do - new_section=$(cfg_get_section $line) - # got a new section - if [[ -n "$new_section" ]]; then - cur_section=$new_section - # not a section, try a key value - else - val=$(cfg_get_key_value $line) - # trim leading and trailing spaces as well - cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') - cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') - if [[ -n "$cur_key" ]]; then - # section + key is the associative in bash array, the field separator is space - repo_cfg[${cur_section} ${cur_key}]=$cur_val - fi - fi - done <$1 -} - -function cfg_print { - for i in "${!repo_cfg[@]}" - do - # split the associative key in to section and key - echo -n "section : $(echo $i | cut -f1 -d ' ');" - echo -n "key : $(echo $i | cut -f2 -d ' ');" - echo "value: ${repo_cfg[$i]}" - done -} - -function cfg_get_value { - section=$1 - key=$2 - echo "${repo_cfg[$section $key]}" -} From 95c65049ac6148fb531841131e431a585c540a16 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 20:31:27 +0100 Subject: [PATCH 11/75] functions to process repository config file and use data in eessi_container.sh --- cfg_files.sh | 87 ++++++++++++++++++++++++++++++++++++++++++---- eessi_container.sh | 64 ++++++++++++++++++++++++++++++++-- 2 files changed, 142 insertions(+), 9 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index c2c549d0bf..f17b893280 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -13,6 +13,7 @@ # -a -> indexed array # -A -> associative array declare -A cfg_repos +declare -A cfg_file_map # functions @@ -46,28 +47,102 @@ function cfg_load { else val=$(cfg_get_key_value $line) # trim leading and trailing spaces as well - cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') - cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) + #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) if [[ -n "$cur_key" ]]; then # section + key is the associative in bash array, the field separator is space - repo_cfg[${cur_section} ${cur_key}]=$cur_val + cfg_repos[${cur_section} ${cur_key}]=$cur_val fi fi done <$1 } function cfg_print { - for i in "${!repo_cfg[@]}" + for i in "${!cfg_repos[@]}" do # split the associative key in to section and key echo -n "section : $(echo $i | cut -f1 -d ' ');" echo -n "key : $(echo $i | cut -f2 -d ' ');" - echo "value: ${repo_cfg[$i]}" + echo "value: ${cfg_repos[$i]}" done } function cfg_get_value { section=$1 key=$2 - echo "${repo_cfg[$section $key]}" + echo "${cfg_repos[$section $key]}" +} + +function cfg_trim_spaces { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< ${1} + else + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' < /dev/stdin + fi +} + +function cfg_trim_quotes { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^"*//' -e 's/"*$//' <<< ${1} + else + sed -e 's/^"*//' -e 's/"*$//' < /dev/stdin + fi +} + +function cfg_trim_curly_brackets { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^{*//' -e 's/}*$//' <<< ${1} + else + sed -e 's/^{*//' -e 's/}*$//' < /dev/stdin + fi +} + +function cfg_get_all_sections { + # first field in keys + # 1. get first field in all keys, 2. filter duplicates, 3. return them as string + declare -A all_sections + for key in "${!cfg_repos[@]}" + do + section=$(echo "$key" | cut -f1 -d' ') + all_sections[${section}]=1 + done + sections= + for sec_key in "${!all_sections[@]}" + do + sections="${sections} ${sec_key}" + done + echo "${sections}" +} + +function cfg_init_file_map { + # strip '{' and '}' from config_map + # split config_map at ',' + # for each item: split at ':' use first as key, second as value + + # reset global variable + cfg_file_map=() + + # expects a string containing the config_map from the cfg file + # trim leading and trailing curly brackets + cm_trimmed=$(cfg_trim_curly_brackets $1) + + # split into elements along ',' + declare -a cm_mappings +# IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}" + + for index in "${!cm_mappings[@]}" + do + # split mapping into key and value + map_key=$(echo ${cm_mappings[index]} | cut -f1 -d':') + map_value=$(echo ${cm_mappings[index]} | cut -f2 -d':') + # trim spaces and double quotes at start and end + tr_key=$(cfg_trim_spaces "${map_key}" | cfg_trim_quotes) + tr_value=$(cfg_trim_spaces "${map_value}" | cfg_trim_quotes) + cfg_file_map[${tr_key}]=${tr_value} + done } diff --git a/eessi_container.sh b/eessi_container.sh index b597b17647..be0bcccab6 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -26,6 +26,7 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/utils.sh +source ${base_dir}/cfg_files.sh # exit codes: bitwise shift codes to allow for combination of exit codes # ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh @@ -44,6 +45,9 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" +# repository cfg file +REPO_CFG_FILE=repos.cfg + # 0. parse args # see example parsing of command line arguments at @@ -131,6 +135,10 @@ while [[ $# -gt 0 ]]; do PREVIOUS_RUN="$2" shift 2 ;; + -r|--repository) + REPOSITORY="$2" + shift 2 + ;; -x|--http-proxy) HTTP_PROXY="$2" export http_proxy=${HTTP_PROXY} @@ -236,6 +244,7 @@ echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." # |-overlay-upper # |-overlay-work # |-home +# |-cfg source ${base_dir}/init/eessi_defaults @@ -269,11 +278,60 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +# set up repository config (always create cfg dir and populate it with info when +# arg -r|--repository is used) +mkdir -p ${EESSI_TMPDIR}/cfg +if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then + # strip "/cvmfs/" from default setting + repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} +else + # TODO implement more flexible specification of repo cfgs + # REPOSITORY => repo-id OR repo-cfg-file (with a single section) OR + # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) + # + # for now, assuming repo-id is defined in config file pointed to + # REPO_CFG_FILE, which is to be copied into the working directory + # (could also become part of the software layer to define multiple + # standard EESSI repositories) + cfg_load ${REPO_CFG_FILE} + + # cfg file should include: repo_name, repo_version, config_bundle, + # map { local_filepath -> container_filepath } + # + # repo_name_domain is the domain part of the repo_name, e.g., + # eessi-hpc.org for pilot.eessi-hpc.org + # + # where config bundle includes the files (-> target location in container) + # - default.local -> /etc/cvmfs/default.local + # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... + # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf + # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), + # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... + # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} + # a directory that contains the public key to access the repository, key + # itself then doesn't need to be BIND mounted + # - ${repo_name_domain}/${repo_name}.pub + # (-> /etc/cvmfs/keys/${repo_name_domain}/${repo_name}.pub + # the public key to access the repository, key itself is BIND mounted + # via directory ${repo_name_domain} + repo_name=$(cfg_get_value ${REPOSITORY} "repo_name") + # derive domain part from repo_name (everything after first '.') + repo_name_domain=${repo_name#*.} + repo_version=$(cfg_get_value ${REPOSITORY} "repo_version") + config_bundle=$(cfg_get_value ${REPOSITORY} "config_bundle") + config_map=$(cfg_get_value ${REPOSITORY} "config_map") + + # convert config_map into associative array cfg_file_map + cfg_init_file_map + declare -p cfg_file_map + + # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and + # define BIND mounts + exit -1 +fi -# 4. set up vars and dirs specific to a scenario -# strip "/cvmfs/" from default setting -repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} +# 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() if [[ "${ACCESS}" == "ro" ]]; then From f349b1ac841e82ea8c1b739ce5c883cc75c308d3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 20:51:59 +0100 Subject: [PATCH 12/75] small fixes and code style improvements --- cfg_files.sh | 30 +++++++++++++++++++----------- eessi_container.sh | 4 ++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index f17b893280..e1694bfc79 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -60,19 +60,19 @@ function cfg_load { } function cfg_print { - for i in "${!cfg_repos[@]}" - do + for index in "${!cfg_repos[@]}" + do # split the associative key in to section and key - echo -n "section : $(echo $i | cut -f1 -d ' ');" - echo -n "key : $(echo $i | cut -f2 -d ' ');" - echo "value: ${cfg_repos[$i]}" - done + echo -n "section : $(echo $index | cut -f1 -d ' ');" + echo -n "key : $(echo $index | cut -f2 -d ' ');" + echo "value: ${cfg_repos[$index]}" + done } function cfg_get_value { - section=$1 - key=$2 - echo "${cfg_repos[$section $key]}" + section=$1 + key=$2 + echo "${cfg_repos[$section $key]}" } function cfg_trim_spaces { @@ -116,7 +116,7 @@ function cfg_get_all_sections { do sections="${sections} ${sec_key}" done - echo "${sections}" + echo "${sections}" | cfg_trim_spaces } function cfg_init_file_map { @@ -129,7 +129,7 @@ function cfg_init_file_map { # expects a string containing the config_map from the cfg file # trim leading and trailing curly brackets - cm_trimmed=$(cfg_trim_curly_brackets $1) + cm_trimmed=$(cfg_trim_curly_brackets "$1") # split into elements along ',' declare -a cm_mappings @@ -146,3 +146,11 @@ function cfg_init_file_map { cfg_file_map[${tr_key}]=${tr_value} done } + +function cfg_print_map { + for index in "${!cfg_file_map[@]}" + do + echo "${index} --> ${cfg_file_map[${index}]}" + done +} + diff --git a/eessi_container.sh b/eessi_container.sh index be0bcccab6..85f1f55077 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -322,8 +322,8 @@ else config_map=$(cfg_get_value ${REPOSITORY} "config_map") # convert config_map into associative array cfg_file_map - cfg_init_file_map - declare -p cfg_file_map + cfg_init_file_map "${config_map}" + cfg_print_map # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and # define BIND mounts From f1a583c22912c0b1a5bc9c218f3a75a6c5246322 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 21:01:38 +0100 Subject: [PATCH 13/75] uncommented reading of config map --- cfg_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfg_files.sh b/cfg_files.sh index e1694bfc79..885ebd0877 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -133,7 +133,7 @@ function cfg_init_file_map { # split into elements along ',' declare -a cm_mappings -# IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}" + IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}" for index in "${!cm_mappings[@]}" do From ff207f3ad866f3b7c815333f8c964a952f6511b2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 10:33:55 +0100 Subject: [PATCH 14/75] preparing config information for custom repository --- eessi_container.sh | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 85f1f55077..1e1ed8e4ce 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -36,7 +36,7 @@ CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) -REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) @@ -185,7 +185,7 @@ fi # PREVIOUS_RUN_ERROR_EXITCODE # TODO (arg -r|--repository) check if repository is known -# REPOSITORY_UNKNOWN_EXITCODE +# REPOSITORY_ERROR_EXITCODE # TODO (arg -x|--http-proxy) check if http proxy is accessible # HTTP_PROXY_ERROR_EXITCODE @@ -246,8 +246,6 @@ echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." # |-home # |-cfg -source ${base_dir}/init/eessi_defaults - # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} @@ -282,6 +280,9 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then + # need to source defaults as late as possible (see other sourcing below) + source ${base_dir}/init/eessi_defaults + # strip "/cvmfs/" from default setting repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} else @@ -325,9 +326,22 @@ else cfg_init_file_map "${config_map}" cfg_print_map - # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and - # define BIND mounts - exit -1 + # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, + # define BIND mounts and override repo name and version + # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg + if [[ ! -r ${config_bundle} ]]; then + fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + fi + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + for src in "${!cfg_file_map[@]}" + do + target=${cfg_file_map[${src}]} + BIND_PATHS="${BIND_PATHS},${src}:${target}" + done + export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} + export EESSI_CVMFS_REPO_OVERRIDE=${repo_name} + # need to source defaults as late as possible (after *_OVERRIDEs) + source ${base_dir}/init/eessi_defaults fi From b2543b9f70a27ad0be01159797d408362c718836 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 10:35:42 +0100 Subject: [PATCH 15/75] fix source location for config files --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1e1ed8e4ce..cc8ab956da 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -336,7 +336,7 @@ else for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE=${repo_name} From 3f161976d2ff1567e6a197bf91562f8d4aa122e6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 14:37:32 +0100 Subject: [PATCH 16/75] using correct repository paths --- eessi_container.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index cc8ab956da..1df3574a6a 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -339,7 +339,7 @@ else BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} - export EESSI_CVMFS_REPO_OVERRIDE=${repo_name} + export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" # need to source defaults as late as possible (after *_OVERRIDEs) source ${base_dir}/init/eessi_defaults fi @@ -349,8 +349,10 @@ fi declare -a EESSI_FUSE_MOUNTS=() if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}" + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + export EESSI_FUSE_MOUNTS fi if [[ "${ACCESS}" == "rw" ]]; then @@ -362,7 +364,8 @@ if [[ "${ACCESS}" == "rw" ]]; then [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" # set environment variables for fuse mounts in Singularity container - EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" @@ -370,7 +373,10 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_PILOT_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") + export EESSI_FUSE_MOUNTS fi From 935517be94c0ea8b4ca5107aa7251da34a5473f4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 19:29:51 +0100 Subject: [PATCH 17/75] make repository cfg file path configurable --- eessi_container.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1df3574a6a..b8867d2d78 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -45,8 +45,9 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" -# repository cfg file -REPO_CFG_FILE=repos.cfg +# repository cfg file, default name (default location: $PWD) +# can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" # 0. parse args @@ -291,10 +292,10 @@ else # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) # # for now, assuming repo-id is defined in config file pointed to - # REPO_CFG_FILE, which is to be copied into the working directory + # EESSI_REPOS_CFG_FILE, which is to be copied into the working directory # (could also become part of the software layer to define multiple # standard EESSI repositories) - cfg_load ${REPO_CFG_FILE} + cfg_load ${EESSI_REPOS_CFG_FILE} # cfg file should include: repo_name, repo_version, config_bundle, # map { local_filepath -> container_filepath } From d28cc696374cb0523a8583a4dcce58899628778b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 20:51:54 +0100 Subject: [PATCH 18/75] short section about eessi_container.sh --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 4d463b1c5b..daf02eebc2 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,22 @@ Environment set up to use EESSI pilot software stack, have fun! [EESSI pilot 2021.12] $ ``` +### Accessing EESSI via a container + +You need Singularity version 3.7 or newer. Then, simply run + +``` +$ ./eessi_container.sh +``` +Once you get presented the prompt `Singularity>` run the above `source` command. + +If you want to build a package for the software repository, simply add the arguments `--access rw`, e.g., full command would be + +``` +$ ./eessi_container.sh --access rw +``` +Note, not all features/arguments listed via `./eessi_container.sh --help` are implemented. + # License The software in this repository is distributed under the terms of the From eca4f9798213ddc275b7496c1923fd8915af906b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 20:54:31 +0100 Subject: [PATCH 19/75] show which features are not yet implemented --- eessi_container.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index b8867d2d78..fafa54193b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -61,14 +61,20 @@ display_help() { echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMAGE - image file or URL defining the container to use" echo " [default: docker://ghcr.io/eessi/build-node:debian10]" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" echo " -h | --help - display this usage information [default: false]" - echo " -i | --info - display configured repositories [default: false]" echo " -l | --host-storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -m | --mode {shell,run} - shell (launch interactive shell)" echo " run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo + echo " If value for --mode is 'run', the SCRIPT provided is executed." + echo + echo " FEATURES/OPTIONS to be implemented:" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" + echo " -i | --info - display configured repositories [default: false]" echo " -p | --previous-run PRUN - init local disk with data from previous run" echo " format is PATH[:TAR/ZIP] where PATH is pointing" echo " to the previously used local disk, and TAR/ZIP" @@ -76,14 +82,10 @@ display_help() { echo " doesn't exist currently; if PATH exists and" echo " a TAR/ZIP is provided an error is reported" echo " [default: not set]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" echo " [default: not set]" echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" echo " [default: not set]" - echo - echo " If value for --mode is 'run', the SCRIPT provided is executed." } # set defaults for command line arguments From 576897821c2eca3c21f7f168804315dec186466e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 5 Jan 2023 19:53:55 +0100 Subject: [PATCH 20/75] adding support for resuming a previous session --- eessi_container.sh | 146 +++++++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 64 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index fafa54193b..9048b1ae7d 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -35,7 +35,7 @@ ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) -PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) +RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) @@ -58,16 +58,22 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" display_help() { echo "usage: $0 [OPTIONS] [SCRIPT]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMAGE - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian10]" - echo " -h | --help - display this usage information [default: false]" - echo " -l | --host-storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -m | --mode {shell,run} - shell (launch interactive shell)" - echo " run (run a script) [default: shell]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian10]" + echo " -h | --help - display this usage information [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp' of previous run)" + echo " and TGZ is the path to a tarball which is unpacked" + echo " the tmp dir stored on the local storage space (see" + echo " option --storage above) [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -75,13 +81,6 @@ display_help() { echo " -d | --dry-run - run script except for executing the container," echo " print information about setup [default: false]" echo " -i | --info - display configured repositories [default: false]" - echo " -p | --previous-run PRUN - init local disk with data from previous run" - echo " format is PATH[:TAR/ZIP] where PATH is pointing" - echo " to the previously used local disk, and TAR/ZIP" - echo " is used to initialize the local disk if PATH" - echo " doesn't exist currently; if PATH exists and" - echo " a TAR/ZIP is provided an error is reported" - echo " [default: not set]" echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" echo " [default: not set]" echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" @@ -93,10 +92,10 @@ ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian10" DRY_RUN=0 INFO=0 -HOST_STORAGE= +STORAGE= MODE="shell" -PREVIOUS_RUN= REPOSITORY="EESSI-pilot" +RESUME= HTTP_PROXY= HTTPS_PROXY= RUN_SCRIPT_AND_ARGS= @@ -117,6 +116,10 @@ while [[ $# -gt 0 ]]; do DRY_RUN=1 shift 1 ;; + -g|--storage) + STORAGE="$2" + shift 2 + ;; -h|--help) display_help exit 0 @@ -125,23 +128,18 @@ while [[ $# -gt 0 ]]; do INFO=1 shift 1 ;; - -l|--host-storage) - HOST_STORAGE="$2" - #EESSI_TMPDIR="$2" - shift 2 - ;; -m|--mode) MODE="$2" shift 2 ;; - -p|--previous-run) - PREVIOUS_RUN="$2" - shift 2 - ;; -r|--repository) REPOSITORY="$2" shift 2 ;; + -u|--resume) + RESUME="$2" + shift 2 + ;; -x|--http-proxy) HTTP_PROXY="$2" export http_proxy=${HTTP_PROXY} @@ -174,7 +172,7 @@ fi # TODO (arg -c|--container) check container (is it a file or URL & access those) # CONTAINER_ERROR_EXITCODE -# TODO (arg -l|--host-storage) check if it exists, if user has write permission, +# TODO (arg -g|--storage) check if it exists, if user has write permission, # if it contains no data, etc. # HOST_STORAGE_ERROR_EXITCODE @@ -183,13 +181,13 @@ if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi -# TODO (arg -p|--previous-run) check if it exists, if user has read permission, -# if it contains data from a previous run -# PREVIOUS_RUN_ERROR_EXITCODE - # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE +# TODO (arg -u|--resume) check if it exists, if user has read permission, +# if it contains data from a previous run +# RESUME_ERROR_EXITCODE + # TODO (arg -x|--http-proxy) check if http proxy is accessible # HTTP_PROXY_ERROR_EXITCODE @@ -206,37 +204,52 @@ if [[ "${MODE}" == "run" ]]; then fi -# 2. set up host storage/tmp -# as location for temporary data use in the following order -# a. command line argument -l|--host-storage -# b. env var TMPDIR -# c. /tmp -# note, we ensure that (a) takes precedence by setting TMPDIR to HOST_STORAGE -# if HOST_STORAGE is not empty -# note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to -# create a temporary directory -# note, if previous run is used the name of the temporary directory -# should be identical to previous run, ie, then we don't create a new -# temporary directory -if [[ ! -z ${HOST_STORAGE} ]]; then - export TMPDIR=${HOST_STORAGE} - # mktemp fails if TMPDIR does not exist, so let's create it - mkdir -p ${TMPDIR} -fi -if [[ ! -z ${TMPDIR} ]]; then - # TODO check if TMPDIR already exists - # mktemp fails if TMPDIR does not exist, so let's create it - mkdir -p ${TMPDIR} -fi -if [[ -z ${TMPDIR} ]]; then - # mktemp falls back to using /tmp if TMPDIR is empty - # TODO check if /tmp is writable, large enough and usable (different - # features for ro-access and rw-access) - echo "skipping sanity checks for /tmp" +# 2. set up host storage/tmp if necessary +# if session to be resumed from a previous one (--resume ARG) and ARG is a directory +# just reuse ARG, define environment variables accordingly and skip creating a new +# tmp storage +if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then + # resume from directory ${RESUME} + # skip creating a new tmp directory, just set environment variables + echo "Resuming from previous run using temporary storage at ${RESUME}" + EESSI_HOST_STORAGE=${RESUME} +else + # we need a tmp location (and possibly init it with ${RESUME} if it was not + # a directory + + # as location for temporary data use in the following order + # a. command line argument -l|--host-storage + # b. env var TMPDIR + # c. /tmp + # note, we ensure that (a) takes precedence by setting TMPDIR to STORAGE + # if STORAGE is not empty + # note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to + # create a temporary directory + if [[ ! -z ${STORAGE} ]]; then + export TMPDIR=${STORAGE} + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} + fi + if [[ ! -z ${TMPDIR} ]]; then + # TODO check if TMPDIR already exists + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} + fi + if [[ -z ${TMPDIR} ]]; then + # mktemp falls back to using /tmp if TMPDIR is empty + # TODO check if /tmp is writable, large enough and usable (different + # features for ro-access and rw-access) + echo "skipping sanity checks for /tmp" + fi + EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) + echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." fi -EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) -echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." +# if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} +if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then + tar xf ${RESUME} -C ${EESSI_HOST_STORAGE} + echo "Resuming from previous run using temporary storage ${RESUME} unpacked into ${EESSI_HOST_STORAGE}" +fi # 3. set up common vars and directories # directory structure should be: @@ -335,7 +348,12 @@ else if [[ ! -r ${config_bundle} ]]; then fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + + # only unpack config_bundle if we're not resuming from a previous run + if [[ -z ${RESUME} ]]; then + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + fi + for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} From bf90a48478f810efbf12e227041069145b5423b9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 5 Jan 2023 20:06:23 +0100 Subject: [PATCH 21/75] improving message for tmp and resume option --- eessi_container.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9048b1ae7d..71efdd0c01 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,10 +70,10 @@ display_help() { echo " repository to use [default: EESSI-pilot]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp' of previous run)" - echo " and TGZ is the path to a tarball which is unpacked" - echo " the tmp dir stored on the local storage space (see" - echo " option --storage above) [default: not set]" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -242,7 +242,7 @@ else echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) - echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." + echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." fi # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} From 6808ec587246a1d92f6fd06cf5e705218d57e001 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 6 Jan 2023 11:24:16 +0100 Subject: [PATCH 22/75] first attempt at implementing --save option --- eessi_container.sh | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 71efdd0c01..e19e44ca91 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -35,8 +35,9 @@ ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) -RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) -REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) +RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) @@ -74,6 +75,11 @@ display_help() { echo " run) and TGZ is the path to a tarball which is" echo " unpacked the tmp dir stored on the local storage space" echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be" + echo " {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -96,6 +102,7 @@ STORAGE= MODE="shell" REPOSITORY="EESSI-pilot" RESUME= +SAVE= HTTP_PROXY= HTTPS_PROXY= RUN_SCRIPT_AND_ARGS= @@ -136,6 +143,10 @@ while [[ $# -gt 0 ]]; do REPOSITORY="$2" shift 2 ;; + -s|--save) + SAVE="$2" + shift 2 + ;; -u|--resume) RESUME="$2" shift 2 @@ -188,6 +199,11 @@ fi # if it contains data from a previous run # RESUME_ERROR_EXITCODE +# TODO (arg -s|--save) check if DIR exists, if user has write permission, +# if TGZ already exists, if user has write permission to directory to which +# TGZ should be written +# SAVE_ERROR_EXITCODE + # TODO (arg -x|--http-proxy) check if http proxy is accessible # HTTP_PROXY_ERROR_EXITCODE @@ -416,3 +432,26 @@ fi echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} + +# 7. save tmp if requested (arg -s|--save) +if [[ ! -z ${SAVE} ]]; then + if [[ -d ${SAVE} ]]; then + # TODO to be implemented; best way a bit unclear + # assume SAVE is name of a directory to which tarball shall be written to + # name format: {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz + # use container to determine OS and ARCH (same approach as in eessi-bot-build.slurm) + #RUN_CMD="python3 /mnt/os_arch_detect/eessi_software_subdir.py ${GENERIC_OPT}" + # only need READONLY part for CVMFS + #singularity run -B ... ${CONTAINER} ${RUN_CMD} 2>&1) + os=foo + arch=bar + ts=$(date +%s) + TGZ=${REPOSITORY}-${os}-${arch}-${ts}.tgz + else + # assume SAVE is the full path to a tarball's name + tar cf ${SAVE} -C ${EESSI_TMPDIR} . + echo "Save contents of '${EESSI_TMPDIR}' to '${SAVE}' (to resume, add '--resume ${SAVE}')" + fi +fi + +# TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From efe93c72c6e782bc08535cb9045ee1474c3e98e2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 6 Jan 2023 11:57:05 +0100 Subject: [PATCH 23/75] added support for providing just a directory to deposit a tarball of tmp --- eessi_container.sh | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e19e44ca91..01638ebf03 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -78,8 +78,8 @@ display_help() { echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" echo " directory DIR or provided with the fixed full path TGZ" echo " when a directory is provided, the format of the" - echo " tarball's name will be" - echo " {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz [default: not set]" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -435,23 +435,21 @@ singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS # 7. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then + # Note, for now we don't try to be smart and record in any way the OS and + # ARCH which might have been used internally, eg, when software packages + # were built ... we rather keep the script here "stupid" and leave the handling + # of these aspects to where the script is used if [[ -d ${SAVE} ]]; then - # TODO to be implemented; best way a bit unclear # assume SAVE is name of a directory to which tarball shall be written to - # name format: {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz - # use container to determine OS and ARCH (same approach as in eessi-bot-build.slurm) - #RUN_CMD="python3 /mnt/os_arch_detect/eessi_software_subdir.py ${GENERIC_OPT}" - # only need READONLY part for CVMFS - #singularity run -B ... ${CONTAINER} ${RUN_CMD} 2>&1) - os=foo - arch=bar + # name format: {REPO_ID}-{TIMESTAMP}.tgz ts=$(date +%s) - TGZ=${REPOSITORY}-${os}-${arch}-${ts}.tgz + TGZ=${SAVE}/${REPOSITORY}-${ts}.tgz else # assume SAVE is the full path to a tarball's name - tar cf ${SAVE} -C ${EESSI_TMPDIR} . - echo "Save contents of '${EESSI_TMPDIR}' to '${SAVE}' (to resume, add '--resume ${SAVE}')" + TGZ=${SAVE} fi + tar cf ${TGZ} -C ${EESSI_TMPDIR} . + echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From f896c51b73272b7b9f3dab6a12b6a700292e0a12 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 6 Jan 2023 12:15:07 +0100 Subject: [PATCH 24/75] cleaning up code, making script more silent by default --- eessi_container.sh | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 01638ebf03..57b003368b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings @@ -87,10 +87,10 @@ display_help() { echo " -d | --dry-run - run script except for executing the container," echo " print information about setup [default: false]" echo " -i | --info - display configured repositories [default: false]" - echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - echo " [default: not set]" - echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - echo " [default: not set]" + #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + #echo " [default: not set]" + #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + #echo " [default: not set]" } # set defaults for command line arguments @@ -103,8 +103,8 @@ MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= -HTTP_PROXY= -HTTPS_PROXY= +#HTTP_PROXY= +#HTTPS_PROXY= RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; - -x|--http-proxy) - HTTP_PROXY="$2" - export http_proxy=${HTTP_PROXY} - shift 2 - ;; - -y|--https-proxy) - HTTPS_PROXY="$2" - export https_proxy=${HTTPS_PROXY} - shift 2 - ;; +# -x|--http-proxy) +# HTTP_PROXY="$2" +# export http_proxy=${HTTP_PROXY} +# shift 2 +# ;; +# -y|--https-proxy) +# HTTPS_PROXY="$2" +# export https_proxy=${HTTPS_PROXY} +# shift 2 +# ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -255,7 +255,7 @@ else # mktemp falls back to using /tmp if TMPDIR is empty # TODO check if /tmp is writable, large enough and usable (different # features for ro-access and rw-access) - echo "skipping sanity checks for /tmp" + [[ ${INFO} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." @@ -356,7 +356,7 @@ else # convert config_map into associative array cfg_file_map cfg_init_file_map "${config_map}" - cfg_print_map + [[ ${INFO} -eq 1 ]] && cfg_print_map # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, # define BIND mounts and override repo name and version @@ -431,7 +431,9 @@ fi echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" -singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level +# provided to the script +singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} # 7. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then From 2e66057724c2a3b51b92e10c11edd910691d8075 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 15 Jan 2023 21:01:17 +0100 Subject: [PATCH 25/75] Fix hash for checkout action in README CI --- .github/workflows/tests_readme.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index 29dedcc3ff..5c6d0318d4 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -18,14 +18,15 @@ jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults - run: | - source init/eessi_defaults - grep "${EESSI_PILOT_VERSION}" README.md + - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults + run: | + source init/eessi_defaults + grep "${EESSI_PILOT_VERSION}" README.md - - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults - run: | - source init/eessi_defaults - grep "${EESSI_CVMFS_REPO}" README.md + - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults + run: | + source init/eessi_defaults + grep "${EESSI_CVMFS_REPO}" README.md From 447bb2d0da7f6b41a149344eb055ed16ca188663 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 24 Jan 2023 21:59:41 +0100 Subject: [PATCH 26/75] added a function to obtain full path to a tool --- utils.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/utils.sh b/utils.sh index 5d8455bb68..cd7f758a0d 100644 --- a/utils.sh +++ b/utils.sh @@ -31,3 +31,35 @@ function check_exit_code { fatal_error "${fail_msg}" fi } + +function get_path_for_tool { + tool_name=$1 + tool_envvar_name=$2 + + which_out=$(which ${tool_name} 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 + echo "${which_out}" + return 0 + fi + if [[ -z "${tool_envvar_name}" ]]; then + msg="no env var holding the full path to tool '${tool_name}' provided" + echo "${msg}" >&2 + return 1 + else + tool_envvar_value=${!tool_envvar_name} + if [[ -x "${tool_envvar_value}" ]]; then + msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" + echo "${msg}" >&2 + echo "${tool_envvar_value}" + return 0 + else + msg="ERROR: tool '${tool_name}' not in PATH\n" + msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" + echo "${msg}" >&2 + echo "" + return 2 + fi + fi +} From e2c24cf757b912aef167364aa34d0abf723b7b03 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 24 Jan 2023 22:01:13 +0100 Subject: [PATCH 27/75] partial bot/build.sh for eessi_container.sh --- bot/build.sh | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 bot/build.sh diff --git a/bot/build.sh b/bot/build.sh new file mode 100755 index 0000000000..90753db8a8 --- /dev/null +++ b/bot/build.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# script to build the EESSI software layer. Intended use is that it is called +# by a (batch) job running on a compute node. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files references in job.cfg +# - the tool 'yq' for working with json files is available via the PATH or +# the environment variable BOT_YQ (see https://github.com/mikefarah/yq) + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# source utils.sh +source utils.sh + +# check setup / define key variables +# get path for 'yq' (if not found, an empty string is returned) +YQ=$(get_path_for_tool "yq" "BOT_YQ") +exit_code=$? +if [[ ${exit_code} -ne 0 ]]; then + fatal_error "could not find path to 'yq'; exiting" +else + echo_green "found yq (${YQ})" +fi + +# check if './cfg/job.cfg' exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "obtaining configuration settings from '${JOB_CFG_FILE}'" + +LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE}) +echo "LOCAL_TMP='${LOCAL_TMP}'" +echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +storage=$(envsubst <<< ${LOCAL_TMP}) +echo "'${storage}'" + + +# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND +LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE}) +echo "LOAD_MODULES='${LOAD_MODULES}'" + +export SINGULARITY_HOME="$(pwd):/eessi_bot_job" +export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" +mkdir -p ${SINGULARITY_TMPDIR} + +if [[ ${storage} != /tmp* ]] ; +then + export SINGULARITY_BIND="${storage}:/tmp" +fi +echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" + +# load modules LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/build.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/build.sh: no modules to be loaded" +fi + From a71f8e4f79d53f5be848d5f7064ce3569c4e0477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 25 Jan 2023 19:37:55 +0100 Subject: [PATCH 28/75] added function to determine IPv4 address --- utils.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils.sh b/utils.sh index cd7f758a0d..db533adc12 100644 --- a/utils.sh +++ b/utils.sh @@ -63,3 +63,12 @@ function get_path_for_tool { fi fi } + +function get_ipv4_address { + hname=$1 + hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + # TODO try other methods if the one above does not work --> tool that verifies + # what method can be used? + echo "${hipv4}" + return 0 +} From b5bf008e4136b7877e06ca0602b6fd6aa49128c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 25 Jan 2023 19:39:53 +0100 Subject: [PATCH 29/75] various updates for bot/build.sh and eessi_container.sh --- bot/build.sh | 69 ++++++++++++++++++++++---- eessi_container.sh | 117 ++++++++++++++++++++++++--------------------- 2 files changed, 121 insertions(+), 65 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 90753db8a8..e7cc7983bb 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -42,30 +42,37 @@ if [[ ! -r "${JOB_CFG_FILE}" ]]; then fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" -LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE}) +# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy +HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) +echo "HTTP_PROXY='${HTTP_PROXY}'" + +LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" -echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> " +# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided + +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " # replace any env variable in ${LOCAL_TMP} with its # current value (e.g., a value that is local to the job) -storage=$(envsubst <<< ${LOCAL_TMP}) -echo "'${storage}'" - +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" -# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND -LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE}) +# obtain list of modules to be loaded +LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) echo "LOAD_MODULES='${LOAD_MODULES}'" +# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND +CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE}) export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} -if [[ ${storage} != /tmp* ]] ; +if [[ ${STORAGE} != /tmp* ]] ; then - export SINGULARITY_BIND="${storage}:/tmp" + export SINGULARITY_BIND="${STORAGE}:/tmp" fi echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" -# load modules LOAD_MODULES is not empty +# load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') do @@ -76,3 +83,45 @@ else echo "bot/build.sh: no modules to be loaded" fi +# determine repository to be used from entry .repository in cfg/job.cfg +REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE}) + +# determine architecture to be used from entry .architecture in cfg/job.cfg +# default: leave empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) + +source init/minimal_eessi_env + +# TODO +# - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh +# - add handling of http(s)_proxy to eessi_container.sh, in there needs the +# CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust +# way to determine the IP address of a proxy) +# - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying +# files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg') + +# prepare options and directories for calling eessi_container.sh +mkdir -p previous_tmp +run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX) +CONTAINER_OPT= +if [[ ! -z ${CONTAINER} ]]; then + CONTAINER_OPT="--container ${CONTAINER}" +fi +HTTP_PROXY_OPT= +if [[ ! -z ${HTTP_PROXY} ]]; then + HTTP_PROXY_OPT="--container ${HTTP_PROXY}" +fi +REPOSITORY_OPT= +if [[ ! -z ${REPOSITORY} ]]; then + REPOSITORY_OPT="--repository ${REPOSITORY}" +fi +./eessi_container.sh --access rw \ + ${CONTAINER_OPT} \ + ${HTTP_PROXY_OPT} \ + --info \ + --mode run \ + ${REPOSITORY_OPT} \ + --save $(pwd)/previous_tmp \ + --storage ${STORAGE} \ + ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr) diff --git a/eessi_container.sh b/eessi_container.sh index 57b003368b..0016c8c9f0 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings @@ -59,43 +59,43 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" display_help() { echo "usage: $0 [OPTIONS] [SCRIPT]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMG - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian10]" - echo " -h | --help - display this usage information [default: false]" - echo " -g | --storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" - echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," - echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp ...' of a previous" - echo " run) and TGZ is the path to a tarball which is" - echo " unpacked the tmp dir stored on the local storage space" - echo " (see option --storage above) [default: not set]" - echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" - echo " directory DIR or provided with the fixed full path TGZ" - echo " when a directory is provided, the format of the" - echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" - echo " [default: not set]" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian11]" + echo " -h | --help - display this usage information [default: false]" + echo " -i | --info - display configuration information [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]; uses env var \$http_proxy if set" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]; uses env var \$https_proxy if set" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo echo " FEATURES/OPTIONS to be implemented:" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" - echo " -i | --info - display configured repositories [default: false]" - #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - #echo " [default: not set]" - #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - #echo " [default: not set]" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" } # set defaults for command line arguments ACCESS="ro" -CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 INFO=0 STORAGE= @@ -103,8 +103,8 @@ MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= -#HTTP_PROXY= -#HTTPS_PROXY= +HTTP_PROXY=${http_proxy:-} +HTTPS_PROXY=${https_proxy:-} RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; -# -x|--http-proxy) -# HTTP_PROXY="$2" -# export http_proxy=${HTTP_PROXY} -# shift 2 -# ;; -# -y|--https-proxy) -# HTTPS_PROXY="$2" -# export https_proxy=${HTTPS_PROXY} -# shift 2 -# ;; + -x|--http-proxy) + HTTP_PROXY="$2" + export http_proxy=${HTTP_PROXY} + shift 2 + ;; + -y|--https-proxy) + HTTPS_PROXY="$2" + export https_proxy=${HTTPS_PROXY} + shift 2 + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -281,26 +281,26 @@ fi # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} -[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR='${EESSI_TMPDIR}'" # configure Singularity export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache mkdir -p ${SINGULARITY_CACHEDIR} -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} mkdir -p ${EESSI_CVMFS_VAR_RUN} -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB='${EESSI_CVMFS_VAR_LIB}'" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN='${EESSI_CVMFS_VAR_RUN}'" # allow that SINGULARITY_HOME is defined before script is run if [[ -z ${SINGULARITY_HOME} ]]; then export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" mkdir -p ${EESSI_TMPDIR}/home - [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" + [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'" fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) @@ -308,9 +308,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" -# set up repository config (always create cfg dir and populate it with info when +# set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) -mkdir -p ${EESSI_TMPDIR}/cfg +mkdir -p ${EESSI_TMPDIR}/repos_cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${base_dir}/init/eessi_defaults @@ -367,13 +367,13 @@ else # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" @@ -397,8 +397,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -427,7 +427,14 @@ if [[ -z ${SINGULARITY_BIND} ]]; then else export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" fi -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" + +# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) + export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +fi echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" From 88423297b8ef4f6de78c0d00e3f6fc6b6d2b4d85 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 27 Jan 2023 12:32:45 +0100 Subject: [PATCH 30/75] add CaDiCaL/1.3.0 with GCC/9.3.0 to NESSI --- EESSI-pilot-install-software.sh | 5 +++++ eessi-2021.12.yml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index e1b4089e2d..f8c9480196 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -430,6 +430,11 @@ $EB SciPy-bundle-2021.05-foss-2021a.eb --robot check_exit_code $? "${ok_msg}" "${fail_msg}" ### add packages here +echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." +ok_msg="CaDiCaL installed. Nice!" +fail_msg="Installation of CaDiCaL failed, that's unexpected..." +$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot +check_exit_code $? "${ok_msg}" "${fail_msg}" echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" diff --git a/eessi-2021.12.yml b/eessi-2021.12.yml index 210bbb2845..25ea2f0f7a 100644 --- a/eessi-2021.12.yml +++ b/eessi-2021.12.yml @@ -67,3 +67,7 @@ software: versions: '3.9.1.1': versionsuffix: -dmpar + CaDiCaL: + toolchains: + GCC-9.3.0: + versions: ['1.3.0'] From 993b4edff6b402af8a2f0a6c1cf22e51ccc71fe6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 27 Jan 2023 12:58:56 +0100 Subject: [PATCH 31/75] comment already installed packages --- EESSI-pilot-install-software.sh | 444 ++++++++++++++++---------------- 1 file changed, 222 insertions(+), 222 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f8c9480196..5b46ca0d42 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -206,228 +206,228 @@ fi echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." -# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, -# see https://github.com/EESSI/software-layer/issues/123 -# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 -ok_msg="Java installed, off to a good (?) start!" -fail_msg="Failed to install Java, woopsie..." -$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install GCC for foss/2020a -export GCC_EC="GCC-9.3.0.eb" -echo ">> Starting slow with ${GCC_EC}..." -ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -fail_msg="Installation of ${GCC_EC} failed!" -# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place -$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install CMake with custom easyblock that patches CMake when --sysroot is used -echo ">> Install CMake with fixed easyblock to take into account --sysroot" -ok_msg="CMake installed!" -fail_msg="Installation of CMake failed, what the ..." -$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 -echo ">> Installing OpenBLAS..." -ok_msg="Done with OpenBLAS!" -fail_msg="Installation of OpenBLAS failed!" -if [[ $GENERIC -eq 1 ]]; then - echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" -else - openblas_include_easyblocks_from_pr='' -fi -$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OpenMPI..." -ok_msg="OpenMPI installed, w00!" -fail_msg="Installation of OpenMPI failed, that's not good..." -$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install Python -echo ">> Install Python 2.7.18 and Python 3.8.2..." -ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" -fail_msg="Installation of Python failed, oh no..." -$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Perl..." -ok_msg="Perl installed, making progress..." -fail_msg="Installation of Perl failed, this never happens..." -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Qt5..." -ok_msg="Qt5 installed, phieuw, that was a big one!" -fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# skip test step when installing SciPy-bundle on aarch64, -# to dance around problem with broken numpy tests; -# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 -echo ">> Installing SciPy-bundle" -ok_msg="SciPy-bundle installed, yihaa!" -fail_msg="SciPy-bundle installation failed, bummer..." -SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb -if [[ "$(uname -m)" == "aarch64" ]]; then - $EB $SCIPY_EC --robot --skip-test-step -else - $EB $SCIPY_EC --robot -fi -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing GROMACS..." -ok_msg="GROMACS installed, wow!" -fail_msg="Installation of GROMACS failed, damned..." -$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! -# 32GB is sufficient to build with 16 cores -echo ">> Installing OpenFOAM (twice!)..." -ok_msg="OpenFOAM installed, now we're talking!" -fail_msg="Installation of OpenFOAM failed, we were so close..." -$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi - -echo ">> Installing R 4.0.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Bioconductor 3.11 bundle..." -ok_msg="Bioconductor installed, enjoy!" -fail_msg="Installation of Bioconductor failed, that's annoying..." -$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing TensorFlow 2.3.1..." -ok_msg="TensorFlow 2.3.1 installed, w00!" -fail_msg="Installation of TensorFlow failed, why am I not surprised..." -$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Horovod 0.21.3..." -ok_msg="Horovod installed! Go do some parallel training!" -fail_msg="Horovod installation failed. There comes the headache..." -$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - - echo ">> Installing code-server 3.7.3..." - ok_msg="code-server 3.7.3 installed, now you can use VS Code!" - fail_msg="Installation of code-server failed, that's going to be hard to fix..." - $EB code-server-3.7.3.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi - -echo ">> Installing RStudio-Server 1.3.1093..." -ok_msg="RStudio-Server installed, enjoy!" -fail_msg="Installation of RStudio-Server failed, might be OS deps..." -$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Spark 3.1.1..." -ok_msg="Spark installed, set off the fireworks!" -fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing IPython 7.15.0..." -ok_msg="IPython installed, launch your Jupyter Notebooks!" -fail_msg="Installation of IPython failed, that's unexpected..." -$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing WRF 3.9.1.1..." -ok_msg="WRF installed, it's getting hot in here!" -fail_msg="Installation of WRF failed, that's unexpected..." -OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing R 4.1.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Nextflow 22.10.1..." -ok_msg="Nextflow installed, the work must flow..." -fail_msg="Installation of Nextflow failed, that's unexpected..." -$EB -r --from-pr 16531 Nextflow-22.10.1.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing EasyBuild 4.5.1..." -ok_msg="EasyBuild v4.5.1 installed" -fail_msg="EasyBuild v4.5.1 failed to install" -$EB --from-pr 14545 --include-easyblocks-from-pr 2805 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 -check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" - -echo ">> Installing SciPy-bundle with foss/2021a..." -ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" -fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 -# which includes patch to fix installation with recent Linux kernel headers -$EB --from-pr 14454 GCCcore-10.3.0.eb --robot -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 -# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set -# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 -# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 -# that includes patch to fix bootstrap problem when using alternate sysroot -$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot -# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 -# which includes a patch to fix installation on POWER -$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -# ignore failing FlexiBLAS tests when building on POWER; -# some tests are failing due to a segmentation fault due to "invalid memory reference", -# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; -# using -fstack-protector-strong -fstack-clash-protection should fix that, -# but it doesn't for some reason when building for ppc64le/generic... -if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure -else - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb -fi - -$EB SciPy-bundle-2021.05-foss-2021a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" +##### install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, +##### see https://github.com/EESSI/software-layer/issues/123 +##### and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 +####ok_msg="Java installed, off to a good (?) start!" +####fail_msg="Failed to install Java, woopsie..." +####$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### install GCC for foss/2020a +####export GCC_EC="GCC-9.3.0.eb" +####echo ">> Starting slow with ${GCC_EC}..." +####ok_msg="${GCC_EC} installed, yippy! Off to a good start..." +####fail_msg="Installation of ${GCC_EC} failed!" +##### pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, +##### which includes patch to fix build of GCC 9.3 when recent kernel headers are in place +####$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### install CMake with custom easyblock that patches CMake when --sysroot is used +####echo ">> Install CMake with fixed easyblock to take into account --sysroot" +####ok_msg="CMake installed!" +####fail_msg="Installation of CMake failed, what the ..." +####$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 +####echo ">> Installing OpenBLAS..." +####ok_msg="Done with OpenBLAS!" +####fail_msg="Installation of OpenBLAS failed!" +####if [[ $GENERIC -eq 1 ]]; then +#### echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." +#### openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" +####else +#### openblas_include_easyblocks_from_pr='' +####fi +####$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing OpenMPI..." +####ok_msg="OpenMPI installed, w00!" +####fail_msg="Installation of OpenMPI failed, that's not good..." +####$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### install Python +####echo ">> Install Python 2.7.18 and Python 3.8.2..." +####ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" +####fail_msg="Installation of Python failed, oh no..." +####$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Perl..." +####ok_msg="Perl installed, making progress..." +####fail_msg="Installation of Perl failed, this never happens..." +##### use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +##### to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +####$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Qt5..." +####ok_msg="Qt5 installed, phieuw, that was a big one!" +####fail_msg="Installation of Qt5 failed, that's frustrating..." +####$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### skip test step when installing SciPy-bundle on aarch64, +##### to dance around problem with broken numpy tests; +##### cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 +####echo ">> Installing SciPy-bundle" +####ok_msg="SciPy-bundle installed, yihaa!" +####fail_msg="SciPy-bundle installation failed, bummer..." +####SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb +####if [[ "$(uname -m)" == "aarch64" ]]; then +#### $EB $SCIPY_EC --robot --skip-test-step +####else +#### $EB $SCIPY_EC --robot +####fi +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing GROMACS..." +####ok_msg="GROMACS installed, wow!" +####fail_msg="Installation of GROMACS failed, damned..." +####$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +##### note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! +##### 32GB is sufficient to build with 16 cores +####echo ">> Installing OpenFOAM (twice!)..." +####ok_msg="OpenFOAM installed, now we're talking!" +####fail_msg="Installation of OpenFOAM failed, we were so close..." +####$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then +#### echo ">> Installing QuantumESPRESSO..." +#### ok_msg="QuantumESPRESSO installed, let's go quantum!" +#### fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" +#### $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot +#### check_exit_code $? "${ok_msg}" "${fail_msg}" +####fi +#### +####echo ">> Installing R 4.0.0 (better be patient)..." +####ok_msg="R installed, wow!" +####fail_msg="Installation of R failed, so sad..." +####$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Bioconductor 3.11 bundle..." +####ok_msg="Bioconductor installed, enjoy!" +####fail_msg="Installation of Bioconductor failed, that's annoying..." +####$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing TensorFlow 2.3.1..." +####ok_msg="TensorFlow 2.3.1 installed, w00!" +####fail_msg="Installation of TensorFlow failed, why am I not surprised..." +####$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Horovod 0.21.3..." +####ok_msg="Horovod installed! Go do some parallel training!" +####fail_msg="Horovod installation failed. There comes the headache..." +####$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then +#### +#### echo ">> Installing code-server 3.7.3..." +#### ok_msg="code-server 3.7.3 installed, now you can use VS Code!" +#### fail_msg="Installation of code-server failed, that's going to be hard to fix..." +#### $EB code-server-3.7.3.eb --robot +#### check_exit_code $? "${ok_msg}" "${fail_msg}" +####fi +#### +####echo ">> Installing RStudio-Server 1.3.1093..." +####ok_msg="RStudio-Server installed, enjoy!" +####fail_msg="Installation of RStudio-Server failed, might be OS deps..." +####$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." +####ok_msg="OSU-Micro-Benchmarks installed, yihaa!" +####fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." +####$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Spark 3.1.1..." +####ok_msg="Spark installed, set off the fireworks!" +####fail_msg="Installation of Spark failed, no fireworks this time..." +####$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing IPython 7.15.0..." +####ok_msg="IPython installed, launch your Jupyter Notebooks!" +####fail_msg="Installation of IPython failed, that's unexpected..." +####$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing WRF 3.9.1.1..." +####ok_msg="WRF installed, it's getting hot in here!" +####fail_msg="Installation of WRF failed, that's unexpected..." +####OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing R 4.1.0 (better be patient)..." +####ok_msg="R installed, wow!" +####fail_msg="Installation of R failed, so sad..." +####$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing Nextflow 22.10.1..." +####ok_msg="Nextflow installed, the work must flow..." +####fail_msg="Installation of Nextflow failed, that's unexpected..." +####$EB -r --from-pr 16531 Nextflow-22.10.1.eb +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." +####ok_msg="OSU-Micro-Benchmarks installed, yihaa!" +####fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." +####$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####echo ">> Installing EasyBuild 4.5.1..." +####ok_msg="EasyBuild v4.5.1 installed" +####fail_msg="EasyBuild v4.5.1 failed to install" +####$EB --from-pr 14545 --include-easyblocks-from-pr 2805 +####check_exit_code $? "${ok_msg}" "${fail_msg}" +#### +####LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 +####check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" +#### +####echo ">> Installing SciPy-bundle with foss/2021a..." +####ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" +####fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." +##### use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 +##### which includes patch to fix installation with recent Linux kernel headers +####$EB --from-pr 14454 GCCcore-10.3.0.eb --robot +##### use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 +##### to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) +####$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 +##### use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set +##### from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 +####$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +##### use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 +##### that includes patch to fix bootstrap problem when using alternate sysroot +####$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot +##### use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 +##### which includes a patch to fix installation on POWER +####$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot +##### ignore failing FlexiBLAS tests when building on POWER; +##### some tests are failing due to a segmentation fault due to "invalid memory reference", +##### see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; +##### using -fstack-protector-strong -fstack-clash-protection should fix that, +##### but it doesn't for some reason when building for ppc64le/generic... +####if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then +#### $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure +####else +#### $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb +####fi +#### +####$EB SciPy-bundle-2021.05-foss-2021a.eb --robot +####check_exit_code $? "${ok_msg}" "${fail_msg}" ### add packages here echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." From b5c07eeb99e77191d818e64788b0d419031cf526 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 27 Jan 2023 20:30:58 +0100 Subject: [PATCH 32/75] provide CVMFS_HTTP_PROXY if necessary --- bot/build.sh | 3 ++- eessi_container.sh | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index e7cc7983bb..a4f54cba3c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -95,7 +95,8 @@ source init/minimal_eessi_env # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# - add handling of http(s)_proxy to eessi_container.sh, in there needs the +# TODO ensure that the bot makes use of that. +# - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) # - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying diff --git a/eessi_container.sh b/eessi_container.sh index 0016c8c9f0..c2b4988162 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -19,8 +19,8 @@ # 2. set up host storage/tmp # 3. set up common vars and directories # 4. set up vars specific to a scenario -# 5. initialize host storage/tmp from previous run if provided -# 6. run container +# 5. run container +# 6. save tmp (if requested) # -. initial settings & exit codes base_dir=$(dirname $(realpath $0)) @@ -276,7 +276,7 @@ fi # |-overlay-upper # |-overlay-work # |-home -# |-cfg +# |-repos_cfg # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} @@ -358,9 +358,9 @@ else cfg_init_file_map "${config_map}" [[ ${INFO} -eq 1 ]] && cfg_print_map - # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, + # define BIND mounts and override repo name and version + # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg if [[ ! -r ${config_bundle} ]]; then fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi @@ -381,6 +381,26 @@ else source ${base_dir}/init/eessi_defaults fi +# if http_proxy is not empty, we assume that the machine accesses internet +# via a proxy. then we need to add CVMFS_HTTP_PROXY to +# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND +# MOUNT if it was not yet in BIND_PATHS) +if [[ ! -z ${http_proxy} ]]; then + # TODO tolerate other formats for proxy URLs, for now assume format is + # http://SOME_HOSTNAME:SOME_PORT + PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// + PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port + HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} + HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ + "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + >> ${EESSI_TMPDIR}/repos_cfg/default.local + cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS + if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + fi +fi # 4. set up vars and dirs specific to a scenario @@ -417,10 +437,7 @@ if [[ "${ACCESS}" == "rw" ]]; then fi -# 5. initialize host storage/tmp from previous run if provided - - -# 6. run container +# 5. run container # final settings if [[ -z ${SINGULARITY_BIND} ]]; then export SINGULARITY_BIND="${BIND_PATHS}" @@ -442,7 +459,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ # provided to the script singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} -# 7. save tmp if requested (arg -s|--save) +# 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then # Note, for now we don't try to be smart and record in any way the OS and # ARCH which might have been used internally, eg, when software packages From 4ff37218793559a6e3a8ea9cef58198e33b7dcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:41:36 +0100 Subject: [PATCH 33/75] add functions to derive hostname and port number from proxy URL --- utils.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/utils.sh b/utils.sh index db533adc12..fb4eff1e3f 100644 --- a/utils.sh +++ b/utils.sh @@ -64,6 +64,30 @@ function get_path_for_tool { fi } +function get_host_from_url { + url=$1 + re="(http|https)://([^/:]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_port_from_url { + url=$1 + re="(http|https)://[^:]+:([0-9]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + function get_ipv4_address { hname=$1 hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) From 7a58fbff63712d81dc7dd3206ae2d35535c58b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:43:27 +0100 Subject: [PATCH 34/75] add function to determine config sections --- cfg_files.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index 885ebd0877..ffd29db50a 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -47,9 +47,7 @@ function cfg_load { else val=$(cfg_get_key_value $line) # trim leading and trailing spaces as well - #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) - #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) if [[ -n "$cur_key" ]]; then # section + key is the associative in bash array, the field separator is space @@ -69,6 +67,20 @@ function cfg_print { done } +function cfg_sections { + declare -A sections + for key in "${!cfg_repos[@]}" + do + # extract section from the associative key + section=$(echo $key | cut -f1 -d ' ') + sections[${section}]=1 + done + for repo in "${!sections[@]}" + do + echo "${repo}" + done +} + function cfg_get_value { section=$1 key=$2 From a5ea7d8aea2c92380839ecbf290c28d278c43835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:44:06 +0100 Subject: [PATCH 35/75] add parameter to list repositories, fix bugs fixed bugs - determining IP address for proxy host was wrong - handling of positional parameters for executing commands with arguments was wrong --- eessi_container.sh | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index c2b4988162..d8d7cb2bdb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -66,6 +66,7 @@ display_help() { echo " -i | --info - display configuration information [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" @@ -99,13 +100,13 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 INFO=0 STORAGE= +LIST_REPOS=0 MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= HTTP_PROXY=${http_proxy:-} HTTPS_PROXY=${https_proxy:-} -RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -135,6 +136,10 @@ while [[ $# -gt 0 ]]; do INFO=1 shift 1 ;; + -l|--list-repos) + LIST_REPOS=1 + shift 1 + ;; -m|--mode) MODE="$2" shift 2 @@ -173,6 +178,17 @@ done set -- "${POSITIONAL_ARGS[@]}" +if [[ ${LIST_REPOS} -eq 1 ]]; then + echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':" + echo " EESSI-pilot [default]" + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + echo " ${repo_id}" + done <<< "${sections}" + exit 0 +fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported @@ -214,8 +230,6 @@ fi if [[ "${MODE}" == "run" ]]; then if [[ $# -eq 0 ]]; then fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}" - else - RUN_SCRIPT_AND_ARGS=$@ fi fi @@ -387,17 +401,20 @@ fi # MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is - # http://SOME_HOSTNAME:SOME_PORT - PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// - PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port - HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} - HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) - echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ - "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + # http://SOME_HOSTNAME:SOME_PORT/ + [[ ${INFO} -eq 1 ]] && echo "http_proxy='${http_proxy}'" + PROXY_HOST=$(get_host_from_url ${http_proxy}) + [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" + PROXY_PORT=$(get_port_from_url ${http_proxy}) + [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi @@ -454,10 +471,10 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then fi echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" # 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then From 6ed3d85ff5f11bca37f838c533320733c3ddaf08 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 14:27:26 +0100 Subject: [PATCH 36/75] fix bash pattern matching operator --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index c2b4988162..0cab47e8fb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -397,7 +397,7 @@ if [[ ! -z ${http_proxy} ]]; then >> ${EESSI_TMPDIR}/repos_cfg/default.local cat ${EESSI_TMPDIR}/repos_cfg/default.local # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi From fd5cc3d339bffd7db248c43be909d195db8a64bd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 14:29:25 +0100 Subject: [PATCH 37/75] various updates to bot/build.sh - added handling of HTTPS_PROXY - moved CONTAINER setting from site_config to repository section (different repositories could require different containers) - added handling of CPU_TARGET and EESSI_SOFTWARE_SUBDIR_OVERRIDE - updated TODOs - added setting of EESSI_REPOS_CFG_FILE_OVERRIDE - bug fixes --- bot/build.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index a4f54cba3c..3d06477a15 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -46,6 +46,10 @@ echo "obtaining configuration settings from '${JOB_CFG_FILE}'" HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) echo "HTTP_PROXY='${HTTP_PROXY}'" +# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy +HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE}) +echo "HTTPS_PROXY='${HTTPS_PROXY}'" + LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided @@ -61,7 +65,7 @@ LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) echo "LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND -CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE}) +CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE}) export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} @@ -89,13 +93,19 @@ EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos. # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used -EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) +if [[ ! -z "${CPU_TARGET}" ]]; then + EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} +else + EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) +fi source init/minimal_eessi_env # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# TODO ensure that the bot makes use of that. +# TODO ensure that the bot makes use of that. (currently sets env var +# CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also +# add it to cfg/job.cfg - .architecture.software_subdir) # - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) @@ -111,18 +121,25 @@ if [[ ! -z ${CONTAINER} ]]; then fi HTTP_PROXY_OPT= if [[ ! -z ${HTTP_PROXY} ]]; then - HTTP_PROXY_OPT="--container ${HTTP_PROXY}" + HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" +fi +HTTPS_PROXY_OPT= +if [[ ! -z ${HTTPS_PROXY} ]]; then + HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" fi REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg +export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ + ${HTTPS_PROXY_OPT} \ --info \ --mode run \ ${REPOSITORY_OPT} \ --save $(pwd)/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr) + ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr} From b2eab080cba313c72e2c72f91429242b1bf04990 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 19:10:40 +0100 Subject: [PATCH 38/75] replace FILE with DIR in EESSI_REPOS_CFG_FILE_OVERRIDE --- bot/build.sh | 8 ++++---- eessi_container.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 3d06477a15..7de3594d41 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -89,7 +89,7 @@ fi # determine repository to be used from entry .repository in cfg/job.cfg REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE}) # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used @@ -131,8 +131,8 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi -# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg -export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg +# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg +export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -142,4 +142,4 @@ export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg ${REPOSITORY_OPT} \ --save $(pwd)/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr} + ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr} diff --git a/eessi_container.sh b/eessi_container.sh index d8d7cb2bdb..4ca494e136 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -47,8 +47,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" # repository cfg file, default name (default location: $PWD) -# can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" # 0. parse args From 7e200a185782727ad61f7c1e78f4b52228300a4d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 19:56:41 +0100 Subject: [PATCH 39/75] fix whitespace issues --- cfg_files.sh | 1 - eessi_container.sh | 2 +- init/eessi_defaults | 1 - utils.sh | 4 ++-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index ffd29db50a..57ea2f7c03 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -165,4 +165,3 @@ function cfg_print_map { echo "${index} --> ${cfg_file_map[${index}]}" done } - diff --git a/eessi_container.sh b/eessi_container.sh index 4ca494e136..3a9fac2955 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -407,7 +407,7 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" PROXY_PORT=$(get_port_from_url ${http_proxy}) [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" - HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local diff --git a/init/eessi_defaults b/init/eessi_defaults index 1b5ce07fb1..f482cbc269 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -10,4 +10,3 @@ export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" - diff --git a/utils.sh b/utils.sh index fb4eff1e3f..d0da95e87f 100644 --- a/utils.sh +++ b/utils.sh @@ -73,7 +73,7 @@ function get_host_from_url { else echo "" return 1 - fi + fi } function get_port_from_url { @@ -85,7 +85,7 @@ function get_port_from_url { else echo "" return 1 - fi + fi } function get_ipv4_address { From d0b49d90f7c4989e78e7a07cda5dc5270e12b0fe Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 20:21:36 +0100 Subject: [PATCH 40/75] only print call to eessi_container.sh --- bot/build.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 7de3594d41..4567f04c2e 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -133,13 +133,15 @@ if [[ ! -z ${REPOSITORY} ]]; then fi # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg -./eessi_container.sh --access rw \ - ${CONTAINER_OPT} \ - ${HTTP_PROXY_OPT} \ - ${HTTPS_PROXY_OPT} \ - --info \ - --mode run \ - ${REPOSITORY_OPT} \ - --save $(pwd)/previous_tmp \ - --storage ${STORAGE} \ - ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr} +echo "./eessi_container.sh --access rw" +echo " ${CONTAINER_OPT}" +echo " ${HTTP_PROXY_OPT}" +echo " ${HTTPS_PROXY_OPT}" +echo " --info" +echo " --mode run" +echo " ${REPOSITORY_OPT}" +echo " --save $(pwd)/previous_tmp" +echo " --storage ${STORAGE}" +echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${run_outerr}" + +exit 0 From 13eaecd3cb08788f881a941c2b1c308344e3a240 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 20:30:22 +0100 Subject: [PATCH 41/75] print env at job start --- bot/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 4567f04c2e..d91dd2f9ee 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -20,6 +20,11 @@ # - the tool 'yq' for working with json files is available via the PATH or # the environment variable BOT_YQ (see https://github.com/mikefarah/yq) +echo "###################################################################" +env +echo "###################################################################" +echo + # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" From 0284a6d718b81ff8713630a385ddf03eda82c36e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 21:08:54 +0100 Subject: [PATCH 42/75] fix handling of HTTP(S)_PROXY vars --- bot/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index d91dd2f9ee..6ad33287c8 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -48,11 +48,13 @@ fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy -HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) +HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE}) +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} echo "HTTP_PROXY='${HTTP_PROXY}'" # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy -HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE}) +HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE}) +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} echo "HTTPS_PROXY='${HTTPS_PROXY}'" LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) From 388a10317b66bbc4d5a26a8ab540da030bcbae70 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 17:28:15 +0100 Subject: [PATCH 43/75] print env before launching container --- bot/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 6ad33287c8..0813be330d 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -140,6 +140,9 @@ if [[ ! -z ${REPOSITORY} ]]; then fi # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg +echo "###################################################################" +env +echo "###################################################################" echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" From 8311da46259d7e240f82c8a8efc621f392fc5374 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 19:31:21 +0100 Subject: [PATCH 44/75] skip source init/minimal_eessi_env --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0813be330d..56aed7eef2 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -106,7 +106,7 @@ else EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) fi -source init/minimal_eessi_env +#source init/minimal_eessi_env # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh From 978c8e41453c8527d76ce0d6c7b72ba8b0858522 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 20:19:47 +0100 Subject: [PATCH 45/75] add sanity check for --repository arg --- eessi_container.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 3a9fac2955..ef3d64ff35 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -193,7 +193,7 @@ fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then - fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" + fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi # TODO (arg -c|--container) check container (is it a file or URL & access those) @@ -205,11 +205,15 @@ fi # (arg -m|--mode) check if MODE is known if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then - fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" + fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE +if [[ ! -z "${REPOSITORY}" -a "${REPOSITORY}" != "EESSI-pilot" -a ! -r ${EESSI_REPOS_CFG_FILE} ]]; then + fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" +fi + # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run From 0c1c6d5e431dd8a339f1846dda5bdc34fef8bc95 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 20:26:04 +0100 Subject: [PATCH 46/75] actually run eessi_container.sh --- bot/build.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 56aed7eef2..13de2c794c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -143,6 +143,8 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg echo "###################################################################" env echo "###################################################################" +echo +echo "Excuting command:" echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" @@ -153,5 +155,15 @@ echo " ${REPOSITORY_OPT}" echo " --save $(pwd)/previous_tmp" echo " --storage ${STORAGE}" echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${run_outerr}" +./eessi_container.sh --access rw \ + ${CONTAINER_OPT} \ + ${HTTP_PROXY_OPT} \ + ${HTTPS_PROXY_OPT} \ + --info \ + --mode run \ + ${REPOSITORY_OPT} \ + --save $(pwd)/previous_tmp \ + --storage ${STORAGE} \ + ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${run_outerr} exit 0 From 11b6c62c13177d05de3f4c1f67fa021a52366ae8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 20:32:59 +0100 Subject: [PATCH 47/75] fix logical AND --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index ef3d64ff35..f78f2789ea 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -210,7 +210,7 @@ fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" -a "${REPOSITORY}" != "EESSI-pilot" -a ! -r ${EESSI_REPOS_CFG_FILE} ]]; then +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" fi From c62989b9a925679e4111073b71d05aacfa94b5b9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 22:46:42 +0100 Subject: [PATCH 48/75] fix handling of repo config dirs and files --- bot/build.sh | 7 +++---- eessi_container.sh | 20 ++++++++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 13de2c794c..0ba48bf471 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -96,7 +96,8 @@ fi # determine repository to be used from entry .repository in cfg/job.cfg REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE}) +export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used @@ -117,7 +118,7 @@ fi # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) # - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying -# files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg') +# files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') # prepare options and directories for calling eessi_container.sh mkdir -p previous_tmp @@ -138,8 +139,6 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi -# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg -export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg echo "###################################################################" env echo "###################################################################" diff --git a/eessi_container.sh b/eessi_container.sh index f78f2789ea..269d345fda 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -48,7 +48,10 @@ CVMFS_VAR_RUN="var-run-cvmfs" # repository cfg file, default name (default location: $PWD) # can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg" +# other repository cfg files in directory, default location: $PWD +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" # 0. parse args @@ -379,13 +382,22 @@ else # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, # define BIND mounts and override repo name and version # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - if [[ ! -r ${config_bundle} ]]; then - fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} + else + config_bundle_path=${config_bundle} + fi + + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" From 0c1cec6925c4b6a551b808fa77b8cdd24b69d190 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 00:08:24 +0100 Subject: [PATCH 49/75] remove one bind mount with /tmp as target --- bot/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 0ba48bf471..efe26231cf 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -79,7 +79,8 @@ mkdir -p ${SINGULARITY_TMPDIR} if [[ ${STORAGE} != /tmp* ]] ; then - export SINGULARITY_BIND="${STORAGE}:/tmp" + echo "skip setting SINGULARITY_BIND=${STORAGE}:/tmp because another location is bind mounted to /tmp in eessi_container.sh" + #export SINGULARITY_BIND="${STORAGE}:/tmp" fi echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" From c9daa371f3b735ae922f3af008ac0e17b7ced8e9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 00:22:20 +0100 Subject: [PATCH 50/75] add a few debug messages to run_in_compat_layer_env.sh --- run_in_compat_layer_env.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index e4850d677d..bee4d14c41 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -1,14 +1,20 @@ #!/bin/bash base_dir=$(dirname $(realpath $0)) +echo "script '$0' before sourcing '${base_dir}/init/eessi_defaults'" +env | grep EESSI_ source ${base_dir}/init/eessi_defaults -BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +echo "script '$0' after sourcing '${base_dir}/init/eessi_defaults'" +env | grep EESSI_ + +BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian11" if [ -z $EESSI_PILOT_VERSION ]; then echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 exit 1 fi EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)" +echo "script '$0' compat layer '${EESSI_COMPAT_LAYER_DIR}'" if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2 exit 1 @@ -19,5 +25,5 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi -echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 95ae2bc34c8bb47d896218e80259d5a9a09c6b11 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 08:52:02 +0100 Subject: [PATCH 51/75] add sourcing of minimal_eessi_env --- bot/build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index efe26231cf..9cc4fe9ab5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -108,7 +108,11 @@ else EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) fi -#source init/minimal_eessi_env +echo "environment BEFORE sourcing init/minimal_eessi_env" +env | grep EESSI_ +source init/minimal_eessi_env +echo "environment AFTER sourcing init/minimal_eessi_env" +env | grep EESSI_ # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh From 5ddcec6e6da0eef9f4a7663e0816d3666b665c7f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 11:06:29 +0100 Subject: [PATCH 52/75] add debug output for EESSI_ env vars --- bot/build.sh | 4 ++-- init/minimal_eessi_env | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 9cc4fe9ab5..275302115c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -108,10 +108,10 @@ else EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) fi -echo "environment BEFORE sourcing init/minimal_eessi_env" +echo "environment BEFORE sourcing '${PWD}/init/minimal_eessi_env'" env | grep EESSI_ source init/minimal_eessi_env -echo "environment AFTER sourcing init/minimal_eessi_env" +echo "environment AFTER sourcing '${PWD}/init/minimal_eessi_env'" env | grep EESSI_ # TODO diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index b7cb7c5e9e..97a3729a88 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -3,11 +3,20 @@ # this script is *sourced*, not executed, so can't rely on $0 to determine path to self # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) +echo "init/minimal_eessi_env: BASH_SOURCE='${BASH_SOURCE}'" +echo "init/minimal_eessi_env: realpath BASH_SOURCE='$(realpath ${BASH_SOURCE})'" +echo "init/minimal_eessi_env: EESSI_INIT_DIR_PATH='${EESSI_INIT_DIR_PATH}'" + +echo "EESSI_ settings BEFORE sourcing eessi_defaults" +env | grep EESSI_ # set up defaults: EESSI_CVMFS_REPO, EESSI_PILOT_VERSION # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults +echo "EESSI_ settings AFTER sourcing eessi_defaults" +env | grep EESSI_ + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_PILOT_VERSION if [[ $(uname -s) == 'Linux' ]]; then From 4a859bd098abe2319b75b3433079c1a5004e6fb7 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 11:18:54 +0100 Subject: [PATCH 53/75] remove sourcing from bot/build.sh, add debug output to eessi_container.sh --- bot/build.sh | 10 +++++----- eessi_container.sh | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 275302115c..474bc36a75 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -108,11 +108,11 @@ else EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) fi -echo "environment BEFORE sourcing '${PWD}/init/minimal_eessi_env'" -env | grep EESSI_ -source init/minimal_eessi_env -echo "environment AFTER sourcing '${PWD}/init/minimal_eessi_env'" -env | grep EESSI_ +#echo "environment BEFORE sourcing '${PWD}/init/minimal_eessi_env'" +#env | grep EESSI_ +#source init/minimal_eessi_env +#echo "environment AFTER sourcing '${PWD}/init/minimal_eessi_env'" +#env | grep EESSI_ # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh diff --git a/eessi_container.sh b/eessi_container.sh index 269d345fda..d0fde78937 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -331,6 +331,8 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) +echo "environment BEFORE setting up repository" +env | grep EESSI_ mkdir -p ${EESSI_TMPDIR}/repos_cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) @@ -410,6 +412,8 @@ else # need to source defaults as late as possible (after *_OVERRIDEs) source ${base_dir}/init/eessi_defaults fi +echo "environment AFTER setting up repository" +env | grep EESSI_ # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to From a5695b8974d9122c8f98340e723505fcfcd85d9e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 12:38:54 +0100 Subject: [PATCH 54/75] fixing tiny typo --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 474bc36a75..9139b916e2 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -148,7 +148,7 @@ echo "###################################################################" env echo "###################################################################" echo -echo "Excuting command:" +echo "Executing command:" echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" From 0d06944305e99b60975d03576cf11a35c37cf255 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 12:41:28 +0100 Subject: [PATCH 55/75] export *_OVERRIDEs when calling startprefix --- run_in_compat_layer_env.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index bee4d14c41..3f752a1da7 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -24,6 +24,12 @@ INPUT=$(echo "$@") if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then + INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +fi echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 45e7e6a320cebd45a1c379cd4728a2363ac7de6b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 13:01:53 +0100 Subject: [PATCH 56/75] bump EB version to what is available in nessi.no/2022.11 --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 408c65ecc7..8c2119ecc9 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -137,7 +137,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -REQ_EB_VERSION='4.5.0' +REQ_EB_VERSION='4.6.2' echo ">> Checking for EasyBuild module..." ml_av_easybuild_out=$TMPDIR/ml_av_easybuild.out From 4a90101fab44ddaceb1ac025793030b09fefc4f9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 13:15:06 +0100 Subject: [PATCH 57/75] export http(s)_proxy settings too --- run_in_compat_layer_env.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 3f752a1da7..37f2587de4 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -30,6 +30,12 @@ fi if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${http_proxy} ]; then + INPUT="export http_proxy=${http_proxy}; ${INPUT}" +fi +if [ ! -z ${https_proxy} ]; then + INPUT="export https_proxy=${https_proxy}; ${INPUT}" +fi echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 770ee8dee6b308da938f8adde76ddcc2059e0340 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 13:45:28 +0100 Subject: [PATCH 58/75] check if EASYBUILD_INSTALLPATH is writable --- EESSI-pilot-install-software.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8c2119ecc9..cbb4df9238 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -122,6 +122,10 @@ else fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" fi +echo "@@ Check if installpath '${EASYBUILD_INSTALLPATH}' is writable @@" +touch ${EASYBUILD_INSTALLPATH}/foo +ls -l ${EASYBUILD_INSTALLPATH} + echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild From 9aaefe68428dbb6db6d787bc67dd5d4e29975fde Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 13:56:12 +0100 Subject: [PATCH 59/75] check contents of /tmp --- EESSI-pilot-install-software.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index cbb4df9238..c6760d5ccd 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -126,6 +126,9 @@ echo "@@ Check if installpath '${EASYBUILD_INSTALLPATH}' is writable @@" touch ${EASYBUILD_INSTALLPATH}/foo ls -l ${EASYBUILD_INSTALLPATH} +echo "@@ Check contents of /tmp @@" +ls -l /tmp + echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild From 8819728fdd90c3ce37cda9dc771f51c24f5f5957 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 13:58:51 +0100 Subject: [PATCH 60/75] list contents of /tmp --- eessi_container.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index d0fde78937..a9dab3fbc4 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -490,6 +490,9 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +echo "Show contents of EESSI_TMPDIR='${EESSI_TMPDIR}'" +find ${EESSI_TMPDIR} -type d + echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level From 0bce2f034e4dd8cfd8ae848740cc031d1e750f05 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 14:07:15 +0100 Subject: [PATCH 61/75] moved checks to after sourcing config --- EESSI-pilot-install-software.sh | 6 +++--- eessi_container.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index c6760d5ccd..4fbe7c3522 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -122,6 +122,9 @@ else fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}" fi +echo ">> Configuring EasyBuild..." +source $TOPDIR/configure_easybuild + echo "@@ Check if installpath '${EASYBUILD_INSTALLPATH}' is writable @@" touch ${EASYBUILD_INSTALLPATH}/foo ls -l ${EASYBUILD_INSTALLPATH} @@ -129,9 +132,6 @@ ls -l ${EASYBUILD_INSTALLPATH} echo "@@ Check contents of /tmp @@" ls -l /tmp -echo ">> Configuring EasyBuild..." -source $TOPDIR/configure_easybuild - echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge diff --git a/eessi_container.sh b/eessi_container.sh index a9dab3fbc4..cd9d81e6f1 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -494,10 +494,10 @@ echo "Show contents of EESSI_TMPDIR='${EESSI_TMPDIR}'" find ${EESSI_TMPDIR} -type d echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" +echo "singularity -q ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" # 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then From e9a60ed38fd5ff2cc04edffb59353b1b1814b1ec Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 31 Jan 2023 14:22:16 +0100 Subject: [PATCH 62/75] do not clean /tmp, show last log --- EESSI-pilot-install-software.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 4fbe7c3522..6180a3d521 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -440,8 +440,11 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." ok_msg="CaDiCaL installed. Nice!" fail_msg="Installation of CaDiCaL failed, that's unexpected..." -$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" +$EB CaDiCaL-1.3.0-GCC-9.3.0.eb --robot --disable-cleanup-tmpdir +exit_code=$? +$EB --last-log +cat $($EB --last-log) +check_exit_code $exit_code "${ok_msg}" "${fail_msg}" echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" From 695d3b8ffa2a98e564f5d2303be98ecbc2accab3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 10:11:59 +0100 Subject: [PATCH 63/75] added easystack file for newer NESSI version (content same as for EESSI/2021.12) --- eessi-2022.11.yml | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 eessi-2022.11.yml diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml new file mode 100644 index 0000000000..25ea2f0f7a --- /dev/null +++ b/eessi-2022.11.yml @@ -0,0 +1,73 @@ +software: + code-server: + toolchains: + SYSTEM: + versions: '3.7.3' + GROMACS: + toolchains: + foss-2020a: + versions: + '2020.1': + versionsuffix: -Python-3.8.2 + '2020.4': + versionsuffix: -Python-3.8.2 + Horovod: + toolchains: + foss-2020a: + versions: + '0.21.3': + versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 + Nextflow: + toolchains: + SYSTEM: + versions: '22.10.1' + OpenFOAM: + toolchains: + foss-2020a: + versions: ['8', 'v2006'] + OSU-Micro-Benchmarks: + toolchains: + gompi-2020a: + versions: ['5.6.3'] + gompi-2021a: + versions: ['5.7.1'] + QuantumESPRESSO: + toolchains: + foss-2020a: + versions: ['6.6'] + R: + toolchains: + foss-2021a: + versions: '4.1.0' + R-bundle-Bioconductor: + toolchains: + foss-2020a: + versions: + '3.11': + versionsuffix: -R-4.0.0 + RStudio-Server: + toolchains: + foss-2020a: + versions: + '1.3.1093': + versionsuffix: -Java-11-R-4.0.0 + SciPy-bundle: + toolchains: + foss-2021a: + versions: ['2021.05'] + TensorFlow: + toolchains: + foss-2020a: + versions: + '2.3.1': + versionsuffix: -Python-3.8.2 + WRF: + toolchains: + foss-2020a: + versions: + '3.9.1.1': + versionsuffix: -dmpar + CaDiCaL: + toolchains: + GCC-9.3.0: + versions: ['1.3.0'] From 19c597406dec5ae15db5f916ccc8b611ebdc5c4e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 10:13:12 +0100 Subject: [PATCH 64/75] added TODO comment for handling of TMPDIR, added explicit exit --- EESSI-pilot-install-software.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 6180a3d521..87a6f59b75 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -457,5 +457,9 @@ $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} $TOPDIR/check_missing_installations.sh +# TODO do not clean up by default (or provide a cmd line arg and when run +# from bot, do not clean up) echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} + +exit 0 From 63c58dbddc160e17f69d6a60c0cae8d82221ecd9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 10:14:16 +0100 Subject: [PATCH 65/75] added more easily to parse information about TMP dir and tgz --- eessi_container.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index cd9d81e6f1..c5486b60f2 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -280,6 +280,7 @@ else fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." + echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" fi # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} @@ -516,6 +517,7 @@ if [[ ! -z ${SAVE} ]]; then fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "RESUME_FROM_TGZ ${TGZ}" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From 4ee34ae9ab3d4a71b5b83528055fff7fc1449a9e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 10:15:12 +0100 Subject: [PATCH 66/75] added creating tarball, improved handling of output of container runs --- bot/build.sh | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 9139b916e2..eb49544fdc 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -126,8 +126,6 @@ fi # files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') # prepare options and directories for calling eessi_container.sh -mkdir -p previous_tmp -run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX) CONTAINER_OPT= if [[ ! -z ${CONTAINER} ]]; then CONTAINER_OPT="--container ${CONTAINER}" @@ -148,7 +146,9 @@ echo "###################################################################" env echo "###################################################################" echo -echo "Executing command:" +mkdir -p previous_tmp +build_outerr=$(mktemp build.outerr.XXXX) +echo "Executing command to build software:" echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" @@ -156,9 +156,9 @@ echo " ${HTTPS_PROXY_OPT}" echo " --info" echo " --mode run" echo " ${REPOSITORY_OPT}" -echo " --save $(pwd)/previous_tmp" +echo " --save ${PWD}/previous_tmp" echo " --storage ${STORAGE}" -echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${run_outerr}" +echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -166,8 +166,37 @@ echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${ru --info \ --mode run \ ${REPOSITORY_OPT} \ - --save $(pwd)/previous_tmp \ + --save ${PWD}/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${run_outerr} + ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr} + +# determine temporary directory to resume from +BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") + +tar_outerr=$(mktemp run.outerr.XXXX) +timestamp=$(date +%s) +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${software_subdir//\//-} ${timestamp}) + +echo "Executing command to create tarball:" +echo "./eessi_container.sh --access rw" +echo " ${CONTAINER_OPT}" +echo " ${HTTP_PROXY_OPT}" +echo " ${HTTPS_PROXY_OPT}" +echo " --info" +echo " --mode run" +echo " ${REPOSITORY_OPT}" +echo " --resume ${BUILD_TMPDIR}" +echo " --save ${PWD}/previous_tmp" +echo " ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh --access rw \ + ${CONTAINER_OPT} \ + ${HTTP_PROXY_OPT} \ + ${HTTPS_PROXY_OPT} \ + --info \ + --mode run \ + ${REPOSITORY_OPT} \ + --resume ${BUILD_TMPDIR} \ + --save ${PWD}/previous_tmp \ + ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From 48059a263751d1adef8ceb6e6202c40670ae9cb9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 10:40:56 +0100 Subject: [PATCH 67/75] removed software not available (yet) in NESSI stack --- eessi-2022.11.yml | 68 ----------------------------------------------- 1 file changed, 68 deletions(-) diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml index 25ea2f0f7a..b4d9001de8 100644 --- a/eessi-2022.11.yml +++ b/eessi-2022.11.yml @@ -1,72 +1,4 @@ software: - code-server: - toolchains: - SYSTEM: - versions: '3.7.3' - GROMACS: - toolchains: - foss-2020a: - versions: - '2020.1': - versionsuffix: -Python-3.8.2 - '2020.4': - versionsuffix: -Python-3.8.2 - Horovod: - toolchains: - foss-2020a: - versions: - '0.21.3': - versionsuffix: -TensorFlow-2.3.1-Python-3.8.2 - Nextflow: - toolchains: - SYSTEM: - versions: '22.10.1' - OpenFOAM: - toolchains: - foss-2020a: - versions: ['8', 'v2006'] - OSU-Micro-Benchmarks: - toolchains: - gompi-2020a: - versions: ['5.6.3'] - gompi-2021a: - versions: ['5.7.1'] - QuantumESPRESSO: - toolchains: - foss-2020a: - versions: ['6.6'] - R: - toolchains: - foss-2021a: - versions: '4.1.0' - R-bundle-Bioconductor: - toolchains: - foss-2020a: - versions: - '3.11': - versionsuffix: -R-4.0.0 - RStudio-Server: - toolchains: - foss-2020a: - versions: - '1.3.1093': - versionsuffix: -Java-11-R-4.0.0 - SciPy-bundle: - toolchains: - foss-2021a: - versions: ['2021.05'] - TensorFlow: - toolchains: - foss-2020a: - versions: - '2.3.1': - versionsuffix: -Python-3.8.2 - WRF: - toolchains: - foss-2020a: - versions: - '3.9.1.1': - versionsuffix: -dmpar CaDiCaL: toolchains: GCC-9.3.0: From c6b3bfb7c350931f9adae89416adf98a4b785dc5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 11:36:17 +0100 Subject: [PATCH 68/75] get pilot version and os type from job.cfg, fix other issues in setting TGZ --- bot/build.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index eb49544fdc..ff6321138c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -100,6 +100,11 @@ REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE}) export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +# determine pilot version to be used from .repository.repo_version in cfg/job.cfg +# TODO better? set EESSI_PILOT_VERSION_OVERRIDE and "source init/eessi_defaults" +EESSI_PILOT_VERSION=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE}) +export EESSI_PILOT_VERSION=${EESSI_PILOT_VERSION:-2021.12} + # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used if [[ ! -z "${CPU_TARGET}" ]]; then @@ -108,6 +113,10 @@ else EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) fi +# get EESSI_OS_TYPE from cfg/job.cfg (default: linux) +EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE}) +export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} + #echo "environment BEFORE sourcing '${PWD}/init/minimal_eessi_env'" #env | grep EESSI_ #source init/minimal_eessi_env @@ -173,9 +182,9 @@ echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${bu # determine temporary directory to resume from BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") -tar_outerr=$(mktemp run.outerr.XXXX) +tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) -export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${software_subdir//\//-} ${timestamp}) +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) echo "Executing command to create tarball:" echo "./eessi_container.sh --access rw" From c08cb6f25612b647a7ef701b1ce669569649a134 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 12:02:21 +0100 Subject: [PATCH 69/75] add potential change to getting pilot version --- create_tarball.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index 56ac8ab7ad..87937908c0 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -19,6 +19,8 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} +# overwrite what we get from command line +# pilot_version=${EESSI_PILOT_VERSION} software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then From 0377774f22cf629a73b3a459d703d950f663bdba Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 13:23:57 +0100 Subject: [PATCH 70/75] add a bit debug output to create_tarball.sh --- create_tarball.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index 87937908c0..48fb83394b 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -13,6 +13,11 @@ pilot_version=$2 cpu_arch_subdir=$3 target_tgz=$4 +echo eessi_tmpdir=$1 +echo pilot_version=$2 +echo cpu_arch_subdir=$3 +echo target_tgz=$4 + tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" @@ -26,11 +31,20 @@ software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_ if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 +else + echo "Software directory ${software_dir} does exist! GOOD!" >&2 fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" +echo overlay_upper_dir="${eessi_tmpdir}/overlay-upper" +echo "ls ${overlay_upper_dir}" +ls ${overlay_upper_dir} software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +echo software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}" +echo "find ${overlay_upper_dir} -type d" +find ${overlay_upper_dir} -type d + if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 From c92ead7af6b705071aa3d657d667fb0b95dfcaf0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 1 Feb 2023 14:07:49 +0100 Subject: [PATCH 71/75] try a different tmp directory --- create_tarball.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/create_tarball.sh b/create_tarball.sh index 48fb83394b..42f69fe75c 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -37,6 +37,8 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" echo overlay_upper_dir="${eessi_tmpdir}/overlay-upper" +overlay_upper_dir="/tmp/overlay-upper" +echo overlay_upper_dir="/tmp/overlay-upper" echo "ls ${overlay_upper_dir}" ls ${overlay_upper_dir} From 0b090178a0aaf07040bbb978e7e7620b39858b3d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 2 Feb 2023 23:09:46 +0100 Subject: [PATCH 72/75] call create_tarball.sh with /tmp, revert changes to script --- bot/build.sh | 2 +- create_tarball.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index ff6321138c..1def8ebccb 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -196,7 +196,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" echo " --save ${PWD}/previous_tmp" -echo " ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " ./create_tarball.sh /tmp ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ diff --git a/create_tarball.sh b/create_tarball.sh index 42f69fe75c..c8ede057d3 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -37,8 +37,8 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" echo overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -overlay_upper_dir="/tmp/overlay-upper" -echo overlay_upper_dir="/tmp/overlay-upper" +#overlay_upper_dir="/tmp/overlay-upper" +#echo overlay_upper_dir="/tmp/overlay-upper" echo "ls ${overlay_upper_dir}" ls ${overlay_upper_dir} From 93a488e184ab50845f235d3d3a78eb9ebb42bf50 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 2 Feb 2023 23:28:42 +0100 Subject: [PATCH 73/75] use /tmp as arg to create_tarball.sh --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 1def8ebccb..44985c41ef 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -206,6 +206,6 @@ echo " ./create_tarball.sh /tmp ${EESSI_PILOT_VERSION} ${EES ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ --save ${PWD}/previous_tmp \ - ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + ./create_tarball.sh /tmp ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From 87fbab66b31b673a5f6278fe62bc24655d7f85c1 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 2 Feb 2023 23:47:44 +0100 Subject: [PATCH 74/75] reversing course on using eessi_tmpdir, mounting it into container and how to provide /tmp inside container --- bot/build.sh | 14 +++++++------- eessi_container.sh | 17 ++++++++++------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 44985c41ef..75d3ef698d 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -77,11 +77,11 @@ export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} -if [[ ${STORAGE} != /tmp* ]] ; -then - echo "skip setting SINGULARITY_BIND=${STORAGE}:/tmp because another location is bind mounted to /tmp in eessi_container.sh" - #export SINGULARITY_BIND="${STORAGE}:/tmp" -fi +#if [[ ${STORAGE} != /tmp* ]] ; +#then +# echo "skip setting SINGULARITY_BIND=${STORAGE}:/tmp because another location is bind mounted to /tmp in eessi_container.sh" +# #export SINGULARITY_BIND="${STORAGE}:/tmp" +#fi echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" # load modules if LOAD_MODULES is not empty @@ -196,7 +196,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" echo " --save ${PWD}/previous_tmp" -echo " ./create_tarball.sh /tmp ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -206,6 +206,6 @@ echo " ./create_tarball.sh /tmp ${EESSI_PILOT_VERSION} ${EES ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ --save ${PWD}/previous_tmp \ - ./create_tarball.sh /tmp ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/eessi_container.sh b/eessi_container.sh index c5486b60f2..565209df94 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -327,7 +327,14 @@ fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${EESSI_TMPDIR}" +# provide a '/tmp' inside the container +if [[ ${EESSI_TMPDIR} != /tmp* ]] ; +then + mkdir -p ${EESSI_TMPDIR}/tmp + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/tmp:/tmp" +fi + [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # set up repository config (always create directory repos_cfg and populate it with info when @@ -451,12 +458,8 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper - EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'" - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -465,8 +468,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_TMPDIR}/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_TMPDIR}/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" export EESSI_PILOT_WRITABLE_OVERLAY From 453aff675f459006d2bb20f915f5eba4df526b0f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 3 Feb 2023 10:14:48 +0100 Subject: [PATCH 75/75] always map EESSI_TMPDIR to /tmp --- bot/build.sh | 16 +++++++--------- eessi_container.sh | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 75d3ef698d..54ea46bc37 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -77,13 +77,6 @@ export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} -#if [[ ${STORAGE} != /tmp* ]] ; -#then -# echo "skip setting SINGULARITY_BIND=${STORAGE}:/tmp because another location is bind mounted to /tmp in eessi_container.sh" -# #export SINGULARITY_BIND="${STORAGE}:/tmp" -#fi -echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" - # load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') @@ -186,6 +179,11 @@ tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) +# value of first parameter to create_tarball.sh TMP_IN_CONTAINER needs to be +# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh +# TODO should we make this a configurable parameter of eessi_container.sh using +# /tmp as default? +TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" @@ -196,7 +194,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" echo " --save ${PWD}/previous_tmp" -echo " ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -206,6 +204,6 @@ echo " ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VER ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ --save ${PWD}/previous_tmp \ - ./create_tarball.sh ${BUILD_TMPDIR} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/eessi_container.sh b/eessi_container.sh index 565209df94..277e83aade 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -46,6 +46,9 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" +# target for tmp used inside container +export TMP_IN_CONTAINER=/tmp + # repository cfg file, default name (default location: $PWD) # can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg" @@ -327,13 +330,13 @@ fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${EESSI_TMPDIR}" # provide a '/tmp' inside the container -if [[ ${EESSI_TMPDIR} != /tmp* ]] ; -then - mkdir -p ${EESSI_TMPDIR}/tmp - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/tmp:/tmp" -fi +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" +#if [[ ${EESSI_TMPDIR} != /tmp* ]] ; +#then +# mkdir -p ${EESSI_TMPDIR}/tmp +# BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/tmp:/tmp" +#fi [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" @@ -468,8 +471,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_TMPDIR}/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_TMPDIR}/overlay-work" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" export EESSI_PILOT_WRITABLE_OVERLAY