diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 2c7622b45a..4735d870b0 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -60,6 +60,11 @@ else fi TMPDIR=$(mktemp -d) +echo "TMPDIR=${TMPDIR}; size=$(df -h ${TMPDIR})" + +# are proxies configured +echo "are proxies defined via http*?" +env | grep -i http echo ">> Setting up environment..." @@ -102,7 +107,7 @@ fi # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables -EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables +EESSI_SILENT=0 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then fatal_error "Failed to determine software subdirectory?!" @@ -147,11 +152,16 @@ if [[ $? -eq 0 ]]; then else echo_yellow ">> No EasyBuild module yet, installing it..." + curl -I https://pypi.org/simple/easybuild/ + EB_TMPDIR=${TMPDIR}/ebtmp - echo ">> Temporary installation (in ${EB_TMPDIR})..." + mkdir -p $EB_TMPDIR + echo ">> Temporary installation (in ${EB_TMPDIR}, $(df -h ${EB_TMPDIR}))..." pip_install_out=${TMPDIR}/pip_install.out pip3 install --prefix $EB_TMPDIR easybuild &> ${pip_install_out} + tail ${pip_install_out} + echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..." export PATH=${EB_TMPDIR}/bin:$PATH export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):$PYTHONPATH diff --git a/create_init_tarball.sh b/create_init_tarball.sh index 3393285a33..4d0a2dacaa 100755 --- a/create_init_tarball.sh +++ b/create_init_tarball.sh @@ -1,6 +1,6 @@ #!/bin/bash -SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/main +SOFTWARE_LAYER_TARBALL_URL=https://github.com/trz42/software-layer/tarball/main set -eo pipefail diff --git a/create_tarball.sh b/create_tarball.sh index 619d6ea2e1..342c5f03b8 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -2,14 +2,80 @@ set -e -if [ $# -ne 4 ]; then - echo "ERROR: Usage: $0 " >&2 +if [ $# -lt 3 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 -pilot_version=$2 -cpu_arch_subdir=$3 -target_tgz=$4 +component=$2 +basedir=$3 + +# see example parsing of command line arguments at +# https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " OPTIONS" + echo " -g | --generic - instructs script to tar files of generic architecture target" + echo " -h | --help - display this usage information" +} + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -g|--generic) + EASYBUILD_OPTARCH="GENERIC" + shift + ;; + -h|--help) + display_help # Call your function + # no shifting needed here, we're done. + exit 0 + ;; + -*|--*) + echo "Error: Unknown option: $1" >&2 + exit 1 + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +TOPDIR=$(dirname $(realpath $0)) + +source $TOPDIR/utils.sh + +# need to source minimal_eessi_env early to have EESSI_CPU_FAMILY defined +source $TOPDIR/init/minimal_eessi_env + +if [ -d $EESSI_CVMFS_REPO ]; then + echo_green "$EESSI_CVMFS_REPO available, OK!" +else + fatal_error "$EESSI_CVMFS_REPO is not available!" +fi + +if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then + echo_yellow ">> Tar'ing GENERIC build, taking appropriate measures!" + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_CPU_FAMILY}/generic +fi + +# if EESSI_SOFTWARE_SUBDIR not set get it (note can be overridden by EESSI_SOFTWARE_SUBDIR_OVERRIDE) +if [ -z $EESSI_SOFTWARE_SUBDIR ]; then + source init/eessi_environment_variables +fi + +cpu_arch_subdir=${EESSI_SOFTWARE_SUBDIR} +cpu_arch_subdir_converted=$(echo ${EESSI_SOFTWARE_SUBDIR} | tr '/' '-') +pilot_version=$EESSI_PILOT_VERSION + +timestamp=$(date +%s) +export target_tgz=$(printf "%s/eessi-%s-%s-%s-%s-%d.tar.gz" ${basedir} ${EESSI_PILOT_VERSION} ${component} ${EESSI_OS_TYPE} ${cpu_arch_subdir_converted} ${timestamp}) tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" @@ -43,13 +109,13 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then # module files - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f >> ${files_list} + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks - find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l >> ${files_list} + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then # installation directories - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* >> ${files_list} + ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} fi topdir=${cvmfs_repo}/versions/ diff --git a/default.local b/default.local new file mode 100644 index 0000000000..bda088a88d --- /dev/null +++ b/default.local @@ -0,0 +1,2 @@ +CVMFS_QUOTA_LIMIT=10000 +CVMFS_HTTP_PROXY=DIRECT diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 10ac1926f4..65f1e681b0 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -18,6 +18,8 @@ function error() { # set up minimal environment: $EESSI_PREFIX, $EESSI_PILOT_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX source $EESSI_INIT_DIR_PATH/minimal_eessi_env +echo "minimal_eessi_env: EESSI_PREFIX=${EESSI_PREFIX}= EESSI_OS_TYPE=${EESSI_OS_TYPE}= EESSI_CPU_FAMILY=${EESSI_CPU_FAMILY}=" >> $output + if [ -d $EESSI_PREFIX ]; then echo "Found EESSI pilot repo @ $EESSI_PREFIX!" >> $output diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index 561a311588..b65980007e 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -1,4 +1,6 @@ #!/bin/bash +echo "is env var EASYBUILD_OPTARCH set?" +env | grep -i EASYBUILD_OPTARCH | sed -e 's/^/run_in_compat_layer.sh:/' if [ -z $EESSI_PILOT_VERSION ]; then echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 exit 1