Pull EESSI/main into NESSI/main

.github/workflows/test_eessi_container_script.yml

@@ -0,0 +1,135 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for eessi_container.sh script
+on: [push, pull_request, workflow_dispatch]
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  eessi_container_script:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        SCRIPT_TEST:
+        - help
+        - listrepos_default
+        - listrepos_custom
+        - run
+        - shell
+        - container
+        - resume
+        # FIXME disabled because '--access rw' is not working in CI environment
+        #- readwrite
+        #- save
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
+
+        - name: install Apptainer
+          run: |
+            ./install_apptainer_ubuntu.sh
+
+        - name: Collect info on test environment
+          run: |
+              mount
+              df -h
+
+        - name: Test eessi_container.sh script
+          run: |
+              test_cmd="cat /etc/os-release"
+              out_pattern="Debian GNU/Linux 11"
+
+              if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then
+                ./eessi_container.sh --help
+
+              # test use of --list-repos without custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
+                outfile=out_listrepos.txt
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+              # test use of --list-repos with custom repos.cfg
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
+                outfile=out_listrepos.txt
+                outfile2=out_listrepos_2.txt
+                mkdir -p ${PWD}/cfg
+                echo "[EESSI/20AB.CD]" > cfg/repos.cfg
+                echo "repo_version = 20AB.CD" >> cfg/repos.cfg
+                echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
+                echo "repo_version = 20HT.TP" >> cfg/repos.cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile}
+                grep "EESSI-pilot" ${outfile}
+
+                export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
+                ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
+                grep "[EESSI/2023.02]" ${outfile2}
+
+              # test use of --mode run
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then
+                outfile=out_run.txt
+                echo "${test_cmd}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --mode shell
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then
+                outfile=out_shell.txt
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --container option, using a totally different container;
+              # cfr. https://github.com/easybuilders/easybuild-containers
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then
+                outfile=out_container.txt
+                container="docker://ghcr.io/eessi/build-node:debian10"
+                ./eessi_container.sh --verbose --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                grep "Debian GNU/Linux 10" ${outfile}
+
+              # test use of '--access rw' to get write access in container
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
+                outfile=out_readwrite.txt
+                fn="test_${RANDOM}.txt"
+                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                chmod u+x test_script.sh
+                export SINGULARITY_BIND="$PWD:/test"
+                ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                # note: must use '--access rw' again here, since touched file is in overlay upper dir
+                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+
+              # test use of --resume
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
+                outfile=out_resume.txt
+                ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" > ${outfile}
+
+                tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
+                rm -f ${outfile}
+
+                # make sure that container image exists
+                test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1)
+
+                ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile}
+                cat ${outfile}
+                grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile}
+                grep "${out_pattern}" ${outfile}
+
+              # test use of --save (+ --resume)
+              elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
+                outfile=out_save.txt
+                fn="test_${RANDOM}.txt"
+                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
+                rm -f ${outfile}
+
+                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
+                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+
+                tar tfv test-save.tar | grep "overlay-upper/${fn}"
+
+              else
+                  echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2
+                  exit 1
+              fi

.github/workflows/tests_scripts.yml

@@ -6,8 +6,9 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 
   pull_request:
     branches:
@@ -16,8 +17,9 @@ on:
       - build_container.sh
       - install_software_layer.sh
       - run_in_compat_layer_env.sh
-      - utils.sh
+      - scripts/utils.sh
       - update_lmod_cache.sh
+      - create_directory_tarballs.sh
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
@@ -27,18 +29,9 @@ jobs:
     - name: checkout
       uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-    # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181
     - name: install Apptainer
       run: |
-        sudo apt-get install alien
-        alien --version
-        apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g')
-        curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm
-        sudo alien -d $apptainer_rpm
-        sudo apt install ./apptainer*.deb
-        apptainer --version
-        # also check whether 'singularity' command is still provided by Apptainer installation
-        singularity --version
+        ./install_apptainer_ubuntu.sh
 
     - name: test install_software_layer.sh script
       run: |
@@ -49,3 +42,13 @@ jobs:
           # force using x86_64/generic, to avoid triggering an installation from scratch
           sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh
           ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh
+
+    - name: test create_directory_tarballs.sh script
+      run: |
+          # scripts need to be copied to /tmp,
+          # since create_directory_tarballs.sh must be accessible from within build container
+          cp -a * /tmp/
+          cd /tmp
+          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12
+          # check if tarballs have been produced
+          ls -l *.tar.gz

EESSI-pilot-install-software.sh

@@ -50,7 +50,7 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))
 
-source $TOPDIR/utils.sh
+source $TOPDIR/scripts/utils.sh
 
 # honor $TMPDIR if it is already defined, use /tmp otherwise
 if [ -z $TMPDIR ]; then

README.md

@@ -19,6 +19,22 @@ Environment set up to use EESSI pilot software stack, have fun!
 [EESSI pilot 2021.12] $
 ```
 
+### Accessing EESSI via a container
+
+You need Singularity version 3.7 or newer. Then, simply run
+
+```
+$ ./eessi_container.sh
+```
+Once you get presented the prompt `Singularity>` run the above `source` command.
+
+If you want to build a package for the software repository, simply add the arguments `--access rw`, e.g., full command would be
+
+```
+$ ./eessi_container.sh --access rw
+```
+Note, not all features/arguments listed via `./eessi_container.sh --help` are implemented.
+
 # License
 
 The software in this repository is distributed under the terms of the

bot/bot-eessi-aws-citc.cfg

@@ -0,0 +1,151 @@
+# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5
+
+[github]
+# replace '123456' with the ID of your GitHub App
+app_id = 281041
+
+# a short (!) name for your app instance that can be used for example
+#   when adding/updating a comment to a PR
+# (!) a short yet descriptive name is preferred because it appears in
+#   comments to the PR
+# for example, the name could include the name of the cluster the bot
+#   runs on and the username which runs the bot
+# NOTE avoid putting an actual username here as it will be visible on
+#      potentially publicly accessible GitHub pages.
+app_name = eessi-bot-citc-aws
+
+# replace '12345678' with the ID of the installation of your GitHub App
+#   (can be derived by creating an event and then checking for the list
+#   of sent events and its payload either via the Smee channel's web page
+#   or via the Advanced section of your GitHub App on github.com)
+installation_id = 33078935
+
+# path to the private key that was generated when the GitHub App was registered
+private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem
+
+
+[buildenv]
+# name of the job script used for building an EESSI stack
+build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm
+
+# The container_cachedir may be used to reuse downloaded container image files
+# across jobs. Thus, jobs can more quickly launch containers.
+container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir
+
+# it may happen that we need to customize some CVMFS configuration
+#   the value of cvmfs_customizations is a dictionary which maps a file
+#   name to an entry that needs to be added to that file
+cvmfs_customizations = {}
+
+# if compute nodes have no internet connection, we need to set http(s)_proxy
+#   or commands such as pip3 cannot download software from package repositories
+#   for example, the temporary EasyBuild is installed via pip3 first
+# http_proxy = http://PROXY_DNS:3128/
+# https_proxy = http://PROXY_DNS:3128/
+
+# directory under which the bot prepares directories per job
+#   structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR
+jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
+
+# configure environment
+#   list of comma-separated modules to be loaded by build_job_script
+#   useful/needed if some tool is not provided as system-wide package
+#   (read by bot and handed over to build_job_script via parameter
+#   --load-modules)
+load_modules =
+
+# PATH to temporary directory on build node ... ends up being used for
+#     for example, EESSI_TMPDIR --> /tmp/$USER/EESSI
+#   escaping variables with '\' delays expansion to the start of the
+#     build_job_script; this can be used for referencing environment
+#     variables that are only set inside a Slurm job
+local_tmp = /tmp/$USER/EESSI
+
+# parameters to be added to all job submissions
+# NOTE do not quote parameter string. Quotes are retained when reading in config and
+#      then the whole 'string' is recognised as a single parameter.
+# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs
+#        to be initialised as if it is for a login shell.
+# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed
+slurm_params = --hold --time=24:0:0
+
+# full path to the job submission command
+submit_command = /usr/bin/sbatch
+
+# which GH account has the permission to trigger the build (by setting
+# the label 'bot:build' (apparently this cannot be restricted on GitHub)
+# if value is left/empty everyone can trigger the build
+# value can be a space delimited list of GH accounts
+build_permission = boegel trz42 bedroge
+
+[architecturetargets]
+# defines both for which architectures the bot will build
+#   and what submission parameters shall be used
+# medium instances (8 cores, 16GB RAM)
+#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
+# larger instances (16 cores, 32GB RAM)
+arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"}
+
+[repo_targets]
+# defines for which repository a arch_target should be build for
+#
+# only building for repository EESSI-pilot
+repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]}
+
+# points to definition of repositories (default EESSI-pilot defined by build container)
+repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles
+
+# configuration for event handler which receives events from a GitHub repository.
+[event_handler]
+# path to the log file to log messages for event handler
+log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log
+
+
+[job_manager]
+# path to the log file to log messages for job manager
+log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log
+
+# directory where job manager stores information about jobs to be tracked
+#   e.g. as symbolic link JOBID -> directory to job
+job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs
+
+# full path to the job status checking command
+poll_command = /usr/bin/squeue
+
+# polling interval in seconds
+poll_interval = 60
+
+# full path to the command for manipulating existing jobs
+scontrol_command = /usr/bin/scontrol
+
+[deploycfg]
+# script for uploading built software packages
+tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging
+
+# URL to S3/minio bucket
+#   if attribute is set, bucket_base will be constructed as follows
+#     bucket_base=${endpoint_url}/${bucket_name}
+#   otherwise, bucket_base will be constructed as follows
+#     bucket_base=https://${bucket_name}.s3.amazonaws.com
+# - The former variant is used for non AWS S3 services, eg, minio, or when
+#   the bucket name is not provided in the hostname (see latter case).
+# - The latter variant is used for AWS S3 services.
+#endpoint_url = URL_TO_S3_SERVER
+
+# bucket name
+bucket_name = eessi-staging
+
+# upload policy: defines what policy is used for uploading built artefacts
+#                to an S3 bucket
+# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible)
+# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH)
+#           only upload the latest built artefact
+# 'once'  : only once upload any built artefact for the build target
+# 'none'  : do not upload any built artefacts
+upload_policy = once
+
+# which GH account has the permission to trigger the deployment (by setting
+# the label 'bot:deploy' (apparently this cannot be restricted on GitHub)
+# if value is left/empty everyone can trigger the deployment
+# value can be a space delimited list of GH accounts
+deploy_permission = boegel trz42 bedroge