Changes/TEST PR for bot/build.sh - add CaDiCaL 1.3.0 GCC 9.3.0 (NESSI)

.github/workflows/tests_readme.yml

@@ -0,0 +1,32 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for consistency of README.md
+on:
+  push:
+    paths:
+      - README.md
+      - init/eessi_defaults
+
+  pull_request:
+    branches:
+      - main
+    paths:
+      - README.md
+      - init/eessi_defaults
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+        - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
+          run: |
+            source init/eessi_defaults
+            grep "${EESSI_PILOT_VERSION}" README.md
+
+        - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
+          run: |
+            source init/eessi_defaults
+            grep "${EESSI_CVMFS_REPO}" README.md

README.md

@@ -19,6 +19,22 @@ Environment set up to use EESSI pilot software stack, have fun!
 [EESSI pilot 2021.12] $
 ```
 
+### Accessing EESSI via a container
+
+You need Singularity version 3.7 or newer. Then, simply run
+
+```
+$ ./eessi_container.sh
+```
+Once you get presented the prompt `Singularity>` run the above `source` command.
+
+If you want to build a package for the software repository, simply add the arguments `--access rw`, e.g., full command would be
+
+```
+$ ./eessi_container.sh --access rw
+```
+Note, not all features/arguments listed via `./eessi_container.sh --help` are implemented.
+
 # License
 
 The software in this repository is distributed under the terms of the

bot/build.sh

@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+#
+# script to build the EESSI software layer. Intended use is that it is called
+# by a (batch) job running on a compute node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files references in job.cfg
+#  - the tool 'yq' for working with json files is available via the PATH or
+#    the environment variable BOT_YQ (see https://github.com/mikefarah/yq)
+
+echo "###################################################################"
+env
+echo "###################################################################"
+echo
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+
+# source utils.sh
+source utils.sh
+
+# check setup / define key variables
+# get path for 'yq' (if not found, an empty string is returned)
+YQ=$(get_path_for_tool "yq" "BOT_YQ")
+exit_code=$?
+if [[ ${exit_code} -ne 0 ]]; then
+    fatal_error "could not find path to 'yq'; exiting"
+else
+    echo_green "found yq (${YQ})"
+fi
+
+# check if './cfg/job.cfg' exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "obtaining configuration settings from '${JOB_CFG_FILE}'"
+
+# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy
+HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE})
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
+echo "HTTP_PROXY='${HTTP_PROXY}'"
+
+# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy
+HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE})
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
+echo "HTTPS_PROXY='${HTTPS_PROXY}'"
+
+LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE})
+echo "LOCAL_TMP='${LOCAL_TMP}'"
+# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
+
+echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+# replace any env variable in ${LOCAL_TMP} with its
+#   current value (e.g., a value that is local to the job)
+STORAGE=$(envsubst <<< ${LOCAL_TMP})
+echo "'${STORAGE}'"
+
+# obtain list of modules to be loaded
+LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE})
+echo "LOAD_MODULES='${LOAD_MODULES}'"
+
+# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
+CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE})
+export SINGULARITY_HOME="$(pwd):/eessi_bot_job"
+export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir"
+mkdir -p ${SINGULARITY_TMPDIR}
+
+# load modules if LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/build.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/build.sh: no modules to be loaded"
+fi
+
+# determine repository to be used from entry .repository in cfg/job.cfg
+REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE})
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE})
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+
+# determine pilot version to be used from .repository.repo_version in cfg/job.cfg
+# TODO better? set EESSI_PILOT_VERSION_OVERRIDE and "source init/eessi_defaults"
+EESSI_PILOT_VERSION=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE})
+export EESSI_PILOT_VERSION=${EESSI_PILOT_VERSION:-2021.12}
+
+# determine architecture to be used from entry .architecture in cfg/job.cfg
+# default: leave empty to let downstream script(s) determine subdir to be used
+if [[ ! -z "${CPU_TARGET}" ]]; then
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET}
+else
+    EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE})
+fi
+
+# get EESSI_OS_TYPE from cfg/job.cfg (default: linux)
+EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE})
+export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
+
+#echo "environment BEFORE sourcing '${PWD}/init/minimal_eessi_env'"
+#env | grep EESSI_
+#source init/minimal_eessi_env
+#echo "environment AFTER sourcing '${PWD}/init/minimal_eessi_env'"
+#env | grep EESSI_
+
+# TODO
+#   - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh
+#     TODO ensure that the bot makes use of that. (currently sets env var
+#     CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also
+#     add it to cfg/job.cfg - .architecture.software_subdir)
+#   - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the
+#     CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust
+#     way to determine the IP address of a proxy)
+#   - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying
+#     files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg')
+
+# prepare options and directories for calling eessi_container.sh
+CONTAINER_OPT=
+if [[ ! -z ${CONTAINER} ]]; then
+    CONTAINER_OPT="--container ${CONTAINER}"
+fi
+HTTP_PROXY_OPT=
+if [[ ! -z ${HTTP_PROXY} ]]; then
+    HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}"
+fi
+HTTPS_PROXY_OPT=
+if [[ ! -z ${HTTPS_PROXY} ]]; then
+    HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}"
+fi
+REPOSITORY_OPT=
+if [[ ! -z ${REPOSITORY} ]]; then
+    REPOSITORY_OPT="--repository ${REPOSITORY}"
+fi
+echo "###################################################################"
+env
+echo "###################################################################"
+echo
+mkdir -p previous_tmp
+build_outerr=$(mktemp build.outerr.XXXX)
+echo "Executing command to build software:"
+echo "./eessi_container.sh --access rw"
+echo "                     ${CONTAINER_OPT}"
+echo "                     ${HTTP_PROXY_OPT}"
+echo "                     ${HTTPS_PROXY_OPT}"
+echo "                     --info"
+echo "                     --mode run"
+echo "                     ${REPOSITORY_OPT}"
+echo "                     --save ${PWD}/previous_tmp"
+echo "                     --storage ${STORAGE}"
+echo "                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}"
+./eessi_container.sh --access rw \
+                     ${CONTAINER_OPT} \
+                     ${HTTP_PROXY_OPT} \
+                     ${HTTPS_PROXY_OPT} \
+                     --info \
+                     --mode run \
+                     ${REPOSITORY_OPT} \
+                     --save ${PWD}/previous_tmp \
+                     --storage ${STORAGE} \
+                     ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}
+
+# determine temporary directory to resume from
+BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //")
+
+tar_outerr=$(mktemp tar.outerr.XXXX)
+timestamp=$(date +%s)
+export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
+
+# value of first parameter to create_tarball.sh TMP_IN_CONTAINER needs to be
+# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh
+# TODO should we make this a configurable parameter of eessi_container.sh using
+# /tmp as default?
+TMP_IN_CONTAINER=/tmp
+echo "Executing command to create tarball:"
+echo "./eessi_container.sh --access rw"
+echo "                     ${CONTAINER_OPT}"
+echo "                     ${HTTP_PROXY_OPT}"
+echo "                     ${HTTPS_PROXY_OPT}"
+echo "                     --info"
+echo "                     --mode run"
+echo "                     ${REPOSITORY_OPT}"
+echo "                     --resume ${BUILD_TMPDIR}"
+echo "                     --save ${PWD}/previous_tmp"
+echo "                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+./eessi_container.sh --access rw \
+                     ${CONTAINER_OPT} \
+                     ${HTTP_PROXY_OPT} \
+                     ${HTTPS_PROXY_OPT} \
+                     --info \
+                     --mode run \
+                     ${REPOSITORY_OPT} \
+                     --resume ${BUILD_TMPDIR} \
+                     --save ${PWD}/previous_tmp \
+                     ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+
+exit 0

build_container.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+base_dir=$(dirname $(realpath $0))
+
 BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian11"
 
 if [ $# -lt 2 ]; then
@@ -39,9 +41,13 @@ if [ -z $SINGULARITY_HOME ]; then
     export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER"
 fi
 
+source ${base_dir}/init/eessi_defaults
+# strip "/cvmfs/" from default setting
+repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//}
+
 # set environment variables for fuse mounts in Singularity container
-export EESSI_PILOT_READONLY="container:cvmfs2 pilot.eessi-hpc.org /cvmfs_ro/pilot.eessi-hpc.org"
-export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/pilot.eessi-hpc.org -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work /cvmfs/pilot.eessi-hpc.org"
+export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}"
+export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/${repo_name} -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work ${EESSI_CVMFS_REPO}"
 
 # pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set)
 if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then