Testing PRs to EESSI/software layer

EESSI-pilot-install-software.sh

@@ -60,6 +60,11 @@ else
 fi
 
 TMPDIR=$(mktemp -d)
+echo "TMPDIR=${TMPDIR}; size=$(df -h ${TMPDIR})"
+
+# are proxies configured
+echo "are proxies defined via http*?"
+env | grep -i http
 
 echo ">> Setting up environment..."
 
@@ -102,7 +107,7 @@ fi
 # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
 # $EESSI_SILENT - don't print any messages
 # $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+EESSI_SILENT=0 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
 
 if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
     fatal_error "Failed to determine software subdirectory?!"
@@ -147,11 +152,16 @@ if [[ $? -eq 0 ]]; then
 else
     echo_yellow ">> No EasyBuild module yet, installing it..."
 
+    curl -I https://pypi.org/simple/easybuild/
+
     EB_TMPDIR=${TMPDIR}/ebtmp
-    echo ">> Temporary installation (in ${EB_TMPDIR})..."
+    mkdir -p $EB_TMPDIR
+    echo ">> Temporary installation (in ${EB_TMPDIR}, $(df -h ${EB_TMPDIR}))..."
     pip_install_out=${TMPDIR}/pip_install.out
     pip3 install --prefix $EB_TMPDIR easybuild &> ${pip_install_out}
 
+    tail ${pip_install_out}
+
     echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..."
     export PATH=${EB_TMPDIR}/bin:$PATH
     export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):$PYTHONPATH

create_init_tarball.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/main
+SOFTWARE_LAYER_TARBALL_URL=https://github.com/trz42/software-layer/tarball/main
 
 set -eo pipefail
 

create_tarball.sh

@@ -2,14 +2,80 @@
 
 set -e
 
-if [ $# -ne 4 ]; then
-    echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <pilot version (example: 2021.03)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
+if [ $# -lt 3 ]; then
+        echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <component (software or compat)> <dir to tarball> <any additional options (example: --generic)>" >&2
     exit 1
 fi
 eessi_tmpdir=$1
-pilot_version=$2
-cpu_arch_subdir=$3
-target_tgz=$4
+component=$2
+basedir=$3
+
+# see example parsing of command line arguments at
+#   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
+#   https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+
+display_help() {
+  echo "usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <component (software or compat)> <dir to tarball> [OPTIONS]"
+  echo "  OPTIONS"
+  echo "  -g | --generic         -  instructs script to tar files of generic architecture target"
+  echo "  -h | --help            -  display this usage information"
+}
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -g|--generic)
+      EASYBUILD_OPTARCH="GENERIC"
+      shift
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/utils.sh
+
+# need to source minimal_eessi_env early to have EESSI_CPU_FAMILY defined
+source $TOPDIR/init/minimal_eessi_env
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
+else
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
+fi
+
+if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
+    echo_yellow ">> Tar'ing GENERIC build, taking appropriate measures!"
+    export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_CPU_FAMILY}/generic
+fi
+
+# if EESSI_SOFTWARE_SUBDIR not set get it (note can be overridden by EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+if [ -z $EESSI_SOFTWARE_SUBDIR ]; then
+    source init/eessi_environment_variables
+fi
+
+cpu_arch_subdir=${EESSI_SOFTWARE_SUBDIR}
+cpu_arch_subdir_converted=$(echo ${EESSI_SOFTWARE_SUBDIR} | tr '/' '-')
+pilot_version=$EESSI_PILOT_VERSION
+
+timestamp=$(date +%s)
+export target_tgz=$(printf "%s/eessi-%s-%s-%s-%s-%d.tar.gz" ${basedir} ${EESSI_PILOT_VERSION} ${component} ${EESSI_OS_TYPE} ${cpu_arch_subdir_converted} ${timestamp})
 
 tmpdir=`mktemp -d`
 echo ">> tmpdir: $tmpdir"
@@ -43,13 +109,13 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     # module files
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f >> ${files_list}
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
     # module symlinks
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l >> ${files_list}
+    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
 fi
 if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
     # installation directories
-    ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* >> ${files_list}
+    ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list}
 fi
 
 topdir=${cvmfs_repo}/versions/

default.local

@@ -0,0 +1,2 @@
+CVMFS_QUOTA_LIMIT=10000
+CVMFS_HTTP_PROXY=DIRECT

init/eessi_environment_variables

@@ -18,6 +18,8 @@ function error() {
 # set up minimal environment: $EESSI_PREFIX, $EESSI_PILOT_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX
 source $EESSI_INIT_DIR_PATH/minimal_eessi_env
 
+echo "minimal_eessi_env: EESSI_PREFIX=${EESSI_PREFIX}= EESSI_OS_TYPE=${EESSI_OS_TYPE}= EESSI_CPU_FAMILY=${EESSI_CPU_FAMILY}=" >> $output
+
 if [ -d $EESSI_PREFIX ]; then
   echo "Found EESSI pilot repo @ $EESSI_PREFIX!" >> $output
 

run_in_compat_layer_env.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
+echo "is env var EASYBUILD_OPTARCH set?"
+env | grep -i EASYBUILD_OPTARCH | sed -e 's/^/run_in_compat_layer.sh:/'
 if [ -z $EESSI_PILOT_VERSION ]; then
     echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
     exit 1