diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index 68ba88ac..94679101 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -21,3 +21,4 @@ jobs: # demote var-naming[no-role-prefix] to warnings, as we only have a single role, # and prefixing all variables in that role with the role name is really ugly args: "--warn-list var-naming[no-role-prefix]" + requirements_file: "ansible/galaxy-requirements.yml" diff --git a/ansible/galaxy-requirements.yml b/ansible/galaxy-requirements.yml new file mode 100644 index 00000000..06b90316 --- /dev/null +++ b/ansible/galaxy-requirements.yml @@ -0,0 +1,8 @@ +# +# Install roles and collections from the default Ansible Galaxy server. +# +--- +collections: + - name: community.general + version: '>=10.7.3' +... diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index 95f72c72..9a1afbf7 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -20,9 +20,9 @@ gentoo_git_repo: https://github.com/gentoo/gentoo.git gentoo_git_commit: 083e38cef302128d595e9f9cfd029ad8f67ec2b7 prefix_required_space: 15 GB prefix_user_defined_trusted_dirs: - - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" - - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" - - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/override" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/nvidia" + - "/cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib/amd" prefix_mask_packages: | # stick to GCC 13.x; using a too recent compiler in the compat layer may complicate stuff in the software layer, # see for example https://github.com/EESSI/software-layer/issues/151 diff --git a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml index 15f88ef7..447d7dcf 100644 --- a/ansible/playbooks/roles/compatibility_layer/handlers/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/handlers/main.yml @@ -1,6 +1,15 @@ # Handlers for this role --- +# Handler for (re)generating locales - name: Generate locales ansible.builtin.command: locale-gen changed_when: true + +# Handler for syncing the Gentoo overlays; +# note that it's currently not used, as we do an explicit sync anyway in the playbook, +# but it could still be useful +- name: Sync overlays + community.general.portage: + sync: 'yes' + verbose: true diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index f728896b..db3c6471 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -29,7 +29,9 @@ selectattr('eclass-overrides', 'equalto', True) | map(attribute='name') | join(' ') }} -- name: Sync the repositories +# We do an explicit sync here (instead of using a handler), +# so we can be sure that the overlays are up-to-date. +- name: Sync the overlays to make sure that they are up to date community.general.portage: sync: 'yes' verbose: true diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml index c69f5ee2..3453b927 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/prefix_configuration.yml @@ -2,6 +2,29 @@ # for instance locale support. --- +# To avoid code duplication, the tasks for updating these portage files could be moved to a separate file, +# which is then included by both prefix_configuration.yml and install_prefix.yml. +- name: "Update package.mask file" + ansible.builtin.copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.mask" + content: "{{ prefix_mask_packages }}" + mode: "0644" + when: prefix_mask_packages is defined and prefix_mask_packages | length > 0 + +- name: "Update package.unmask file" + ansible.builtin.copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.unmask" + content: "{{ prefix_unmask_packages }}" + mode: "0644" + when: prefix_unmask_packages is defined and prefix_unmask_packages | length > 0 + +- name: "Update package.use file" + ansible.builtin.copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.use" + content: "{{ prefix_bootstrap_use_flags }}" + mode: "0644" + when: prefix_bootstrap_use_flags is defined and prefix_bootstrap_use_flags | length > 0 + - name: Add locales to configuration file ansible.builtin.lineinfile: path: "{{ gentoo_prefix_path }}/etc/locale.gen" diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml index 36931898..4064592d 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/reprod.yml @@ -1,9 +1,18 @@ -# Store some information and scripts that were used for this installation. +# Store some information and scripts that were used for this installation +# in $EPREFIX/reprod/YYYYMMDD_hhmmssUTC. +# The timestamped subdir ensures that updates of the compatibility layer +# don't overwrite these files; instead they end up in their own subdir. --- -- name: Make a subdirectory for storing build information +- name: Determine timestamped reprod directory for storing information about this build + ansible.builtin.set_fact: + reprod_dir: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ '%Y%m%d_%H%M%SUTC' | strftime(ansible_facts.date_time.epoch | int, utc=True) }}" + tags: + - reprod + +- name: Make a timestamped subdirectory for storing build information ansible.builtin.file: - path: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}" + path: "{{ reprod_dir }}" state: directory mode: '0755' tags: @@ -12,10 +21,12 @@ - name: Copy the used bootstrap script ansible.builtin.copy: src: "{{ prefix_use_builtin_bootstrap | ternary('/usr/local/bin/bootstrap-prefix.sh', prefix_custom_bootstrap_script.remote) }}" - dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/bootstrap-prefix.sh" + dest: "{{ reprod_dir }}/bootstrap-prefix.sh" mode: '0644' tags: - reprod + # Skip this step for updates, as these don't use the bootstrap script anyway + when: not startprefix.stat.exists - name: Get list of installed packages ansible.builtin.command: "qlist -IRv" @@ -27,7 +38,7 @@ - name: Dump list of installed packages to a file ansible.builtin.copy: content: "{{ qlist.stdout }}" - dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ prefix_packages_file }}" + dest: "{{ reprod_dir }}/{{ prefix_packages_file }}" mode: '0644' tags: - reprod @@ -35,7 +46,7 @@ - name: Store other metadata of build in a json file ansible.builtin.copy: content: "{{ metadata | to_nice_json }}" - dest: "{{ gentoo_prefix_path }}/{{ prefix_reprod_dir }}/{{ prefix_metadata_json }}" + dest: "{{ reprod_dir }}/{{ prefix_metadata_json }}" mode: '0644' vars: metadata: diff --git a/bot/build.sh b/bot/build.sh index d59e1505..bf1bbcdc 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -85,27 +85,51 @@ job_repo=$(cfg_get_value "repository" "repo_name") eessi_repo=${job_repo:-software.eessi.io} tar_topdir=/cvmfs/${eessi_repo}/versions -if [ "${eessi_arch}" != "${host_arch}" ]; then - echo "Requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!" +# Check the architecture, which has to match the architecture of the build host +if [[ "${eessi_arch}" != "${host_arch}" ]]; then + echo "bot/build.sh: requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!" exit 1 fi -# option -k is used for retaining ${eessi_tmp} -# store output in local file such that the temporary directory ${STORAGE}/eessi.XXXXXXXXXX -# can be determined +# Read and check the build type (new or update, case-insensitive) +if [[ ! -f bot/build_type ]]; then + echo 'bot/build.sh: cannot determine build type, please add a file "bot/build_type" containing either "new" or "update".' + exit 1 +fi +build_type=$(cat bot/build_type | tr '[:upper:]' '[:lower:]') +if [[ "${build_type}" != "new" && "${build_type}" != "update" ]]; then + echo 'bot/build.sh: invalid build type! It has to be "new" or "update".' + exit 1 +fi +echo "bot/build.sh: requested build type '${build_type}'" + +# Set up the script arguments and run the installation script script_out="install_stdout.log" -./install_compatibility_layer.sh -a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k --verbose 2>&1 | tee -a ${script_out} - -# TODO handle errors (no outfile, no tmp directory found) -eessi_tmp=$(cat ${script_out} | grep 'To resume work add' | cut -f 2 -d \' | cut -f 2 -d ' ') -eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) -# create tarball -> should go into a separate script when this is supported by the bot -target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz -if [ -d ${eessi_tmp}${tar_topdir}/${eessi_version} ]; then - echo ">> Creating tarball ${target_tgz} from ${eessi_tmp}${tar_topdir}..." - tar cfvz ${target_tgz} -C ${eessi_tmp}${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} - echo ${target_tgz} created! -else - echo "Directory ${eessi_tmp}${tar_topdir}/${eessi_version} was not created, not creating tarball." - exit 1 +script_args=(-a ${eessi_arch} -r ${eessi_repo} -g ${STORAGE} -k) +# for (very!) verbose output, uncomment: +#script_args+=(--verbose) +if [[ "${build_type}" == "update" ]]; then + script_args+=(-u) +fi +./install_compatibility_layer.sh ${script_args[@]} 2>&1 | tee -a ${script_out} + +# Create a tarball of the new or updated build +# TODO handle errors (no outfile, no tmp directory found, etc) +eessi_tmp=$(grep '^To resume work add' ${script_out} | cut -f 2 -d \' | cut -f 2 -d ' ' | tail -n 1) +echo "bot/build.sh: creating compatibility layer tarball..." +if [[ "${build_type}" == "update" ]]; then + # Resume the build container session and tar the entire /cvmfs/$repo tree + container_tmp=$(grep -oP '(?<=^Using ).*(?= as tmp directory)' ${script_out} | tail -n 1) + container_image=$(ls -1t ${container_tmp}/*.sif | head -n 1) + # Find the EESSI version by checking which version was changed by the installation script (only one version could have been changed) + eessi_version=$(ls -1 ${container_tmp}/${eessi_repo}/overlay-upper/versions) + target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz + # note: "--access rw" is important, as we need to (re)use the overlay! + ${eessi_tmp}/software-layer-scripts/eessi_container.sh -c ${container_image} --mode exec --resume ${container_tmp} -r ${eessi_repo} -b ${PWD}:/eessi_job --access rw -- tar cfvz /eessi_job/${target_tgz} -C ${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} +elif [[ "${build_type}" == "new" ]]; then + # For a new build, we simply tar the used host directory that was bind mounted as /cvmfs/$repo + # Find the EESSI version by checking which version was created by the installation script in the host directory + eessi_version=$(ls -1 ${eessi_tmp}${tar_topdir}) + target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz + tar cfvz ${target_tgz} -C ${eessi_tmp}${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} fi diff --git a/bot/build_type b/bot/build_type new file mode 100644 index 00000000..4ea5e4dd --- /dev/null +++ b/bot/build_type @@ -0,0 +1 @@ +update diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh index ac70271d..026b25fb 100755 --- a/install_compatibility_layer.sh +++ b/install_compatibility_layer.sh @@ -5,17 +5,18 @@ # ARCH= -CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian11 +CONTAINER=docker://ghcr.io/eessi/build-node-compat-layer:debian-12 REPOSITORY="software.eessi.io" RESUME= RETAIN_TMP=0 STORAGE= +UPDATE=0 VERSION= VERBOSE= # Debian 11 does not support RISC-V, so we use a Debian 13 container instead. if [[ $(uname -m) = "riscv64" ]]; then - CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian-13 + CONTAINER=docker://ghcr.io/eessi/build-node-compat-layer:debian-13 fi display_help() { @@ -46,6 +47,11 @@ display_help() { echo " -t | --resume TMPDIR" echo " tmp directory to resume from [default: None]" echo "" + echo " -u | --update" + echo " update an existing compatibility layer by" + echo " doing a fuse mount of the given repository" + echo " [default: not set]" + echo "" echo " -v | --version VERSION" echo " override the EESSI stack version set in Ansible's" echo " defaults/main.yml file [default: None]" @@ -87,6 +93,10 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; + -u|--update) + UPDATE=1 + shift + ;; -v|--version) VERSION="$2" shift 2 @@ -116,9 +126,6 @@ if [ ! -f "${SCRIPT_DIR}/ansible/playbooks/install.yml" ]; then exit 1 fi -# source utils.sh (for get_container_runtime and check_exit_code) -source ${SCRIPT_DIR}/scripts/utils.sh - # Check if the target architecture is set to the architecture of the current host, # as that's the only thing that's currently supported by this script HOST_ARCH=$(uname -m) @@ -129,7 +136,6 @@ fi if [[ -z ${ARCH} ]]; then ARCH=${HOST_ARCH} fi -echo "A compatibility layer for architecture ${ARCH} will be built." # Make a temporary directory on the host for storing the installation and some temporary files if [[ ! -z ${RESUME} ]] && [[ -d ${RESUME} ]]; then @@ -143,40 +149,12 @@ else fi echo "Using $EESSI_TMPDIR as temporary storage..." -# Create temporary directories -mkdir -p ${EESSI_TMPDIR}/cvmfs -mkdir -p ${EESSI_TMPDIR}/home -mkdir -p ${EESSI_TMPDIR}/tmp - -RUNTIME=$(get_container_runtime) -exit_code=$? -[[ ${VERBOSE} == '-vvv' ]] && echo "RUNTIME='${RUNTIME}'" -check_exit_code ${exit_code} "using runtime ${RUNTIME}" "oh no, neither apptainer nor singularity available" - -# Set up paths and mount points for Apptainer -if [[ -z ${APPTAINER_CACHEDIR} ]]; then - export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache - [[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_CACHEDIR='${APPTAINER_CACHEDIR}'" -fi -export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" -export APPTAINER_BIND="${APPTAINER_BIND},${EESSI_TMPDIR}/tmp:/tmp" -[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_BIND='${APPTAINER_BIND}'" -export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}" -[[ ${VERBOSE} == '-vvv' ]] && echo "APPTAINER_HOME='${APPTAINER_HOME}'" - -# also define SINGULARITY_* env vars -if [[ -z ${SINGULARITY_CACHEDIR} ]]; then - export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache - [[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" -fi -export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" -export SINGULARITY_BIND="${SINGULARITY_BIND},${EESSI_TMPDIR}/tmp:/tmp" -[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" -export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" -[[ ${VERBOSE} == '-vvv' ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'" +# Clone the EESSI/software-layer-scripts repository +git clone https://github.com/EESSI/software-layer-scripts ${EESSI_TMPDIR}/software-layer-scripts +#cp ../eessi_container.sh ${EESSI_TMPDIR}/software-layer-scripts/eessi_container.sh # Construct the Ansible playbook command -ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=$(uname -m)" +ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=${ARCH}" if [[ ! -z ${VERSION} ]]; then ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} -e eessi_version=${VERSION}" fi @@ -187,12 +165,30 @@ if [[ ! -z ${VERBOSE} ]]; then ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} ${VERBOSE}" fi ANSIBLE_COMMAND="ansible-playbook ${ANSIBLE_OPTIONS} /compatibility-layer/ansible/playbooks/install.yml" + +# Set the options for the EESSI container script +CONTAINER_OPTIONS="-c ${CONTAINER} -g ${EESSI_TMPDIR}" +if [[ $UPDATE -eq 0 ]]; then + # For a new compatibility layer, we bind mount an empty host directory as /cvmfs. + # This is a lot faster than (unnecessarily) using an overlay on top of a fuse-mounted /cvmfs. + mkdir "${EESSI_TMPDIR}/cvmfs" + CONTAINER_OPTIONS="${CONTAINER_OPTIONS} -r none -b ${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" +else + # To update an existing compatibility layer, we do have to use an overlay. + CONTAINER_OPTIONS="${CONTAINER_OPTIONS} --access rw -r ${REPOSITORY} -b ${SCRIPT_DIR}:/compatibility-layer" +fi + # Finally, run Ansible inside the container to do the actual installation echo "Executing ${ANSIBLE_COMMAND} in ${CONTAINER}, this will take a while..." -${RUNTIME} shell ${CONTAINER} <