diff --git a/.github/workflows/build-and-publish-bootstrap-container.yml b/.github/workflows/build-and-publish-bootstrap-container.yml index ea5a2272..ba7ef495 100644 --- a/.github/workflows/build-and-publish-bootstrap-container.yml +++ b/.github/workflows/build-and-publish-bootstrap-container.yml @@ -4,14 +4,14 @@ on: branches: - main paths: - - Dockerfile.bootstrap-prefix-centos8 + - Dockerfile.bootstrap-prefix-debian11 - bootstrap-prefix.sh pull_request: branches: - main paths: - - Dockerfile.bootstrap-prefix-centos8 + - Dockerfile.bootstrap-prefix-debian11 - bootstrap-prefix.sh # Declare default permissions as read only. @@ -48,7 +48,7 @@ jobs: - name: Build and push to GitHub Packages uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5 #v3.2.0 with: - tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/bootstrap-prefix:centos8 - file: Dockerfile.bootstrap-prefix-centos8 + tags: ghcr.io/${{ env.REPOSITORY_OWNER }}/bootstrap-prefix:debian11 + file: Dockerfile.bootstrap-prefix-debian11 platforms: linux/amd64, linux/arm64, linux/ppc64le push: ${{ github.event_name != 'pull_request' }} diff --git a/Dockerfile.bootstrap-prefix-centos8 b/Dockerfile.bootstrap-prefix-centos8 deleted file mode 100644 index 0b69df62..00000000 --- a/Dockerfile.bootstrap-prefix-centos8 +++ /dev/null @@ -1,11 +0,0 @@ -FROM centos:8 - -COPY bootstrap-prefix.sh /usr/local/bin/bootstrap-prefix.sh - -RUN dnf install -y gcc gcc-c++ make diffutils gmp-devel perl -RUN chmod 755 /usr/local/bin/bootstrap-prefix.sh - -ENV LC_ALL=C -ENV PATH=/usr/local/bin:$PATH - -ENTRYPOINT ["/usr/local/bin/bootstrap-prefix.sh"] diff --git a/Dockerfile.bootstrap-prefix-debian11 b/Dockerfile.bootstrap-prefix-debian11 new file mode 100644 index 00000000..6c564328 --- /dev/null +++ b/Dockerfile.bootstrap-prefix-debian11 @@ -0,0 +1,17 @@ +FROM debian:11-slim + +COPY bootstrap-prefix.sh /usr/local/bin/bootstrap-prefix.sh + +RUN apt-get update +RUN apt-get install -y gcc g++ make diffutils libgmp-dev perl wget +RUN apt-get install -y git python3-pip python3-cryptography +RUN pip3 install --upgrade pip +RUN pip3 install ansible +RUN chmod 755 /usr/local/bin/bootstrap-prefix.sh + +ENV LC_ALL=C.UTF-8 +ENV PATH=/usr/local/bin:$PATH + +RUN groupadd -g 1000 portage && useradd -ms /bin/bash -u 1000 -g portage portage + +ENTRYPOINT ["/usr/local/bin/bootstrap-prefix.sh"] diff --git a/README.md b/README.md index 747c96fc..d657fbdf 100644 --- a/README.md +++ b/README.md @@ -8,124 +8,15 @@ An alternative would be the [NixOS](https://nixos.org/). ## Installation and Configuration -### Prerequisites - -The bootstrap process will need a clean environment with C and C++ compilers (the system version of gcc and g++ will do) as well as the `make` command. It also is very sensitive to -the environment, so setup a user with unset `CFFLAGS`, `CFLAGS`, `LDFLAGS`, `PKG_CONFIG_PATH` and the always harmful `LD_LIBRARY_PATH` variables. - -EESSI provides a Docker container for this, which can be run via Singularity. - -### Bootstrapping Gentoo Prefix -Gentoo Prefix provides a bootstrap script to build the prefix, see [Gentoo Prefix Bootstrap](https://wiki.gentoo.org/wiki/Project:Prefix/Bootstrap). -We forked [this version](https://gitweb.gentoo.org/repo/proj/prefix.git/tree/scripts/bootstrap-prefix.sh?id=e77fd01734f21ec2e9c985c28ba4eb30c1b2bc9d) -and made some modifications. See issue [#8](https://github.com/EESSI/compatibility-layer/issues/8) for more details. - -You can run our version of the bootstrap script (see `bootstrap-prefix.sh`) inside the Singularity container by executing: -``` -singularity run docker://ghcr.io/eessi/bootstrap-prefix:centos8 -``` - -If you want to run your own version of the bootstrap script, use: -``` -singularity exec docker://ghcr.io/eessi/bootstrap-prefix:centos8 ./bootstrap-prefix.sh -``` -Our version of the script allows you to pick a custom snapshot for the Portage tree. This can be done by setting `CUSTOM_SNAPSHOT_URL` to -a URL that points to a directory, and setting `CUSTOM_SNAPSHOT_URL` to the name of a snapshot file (must be a bzip2 archive). For instance: -``` -env CUSTOM_SNAPSHOT_URL="http://cvmfs-s0.eessi-hpc.org/snapshots" CUSTOM_SNAPSHOT_VERSION="20201209" singularity run docker://ghcr.io/eessi/bootstrap-prefix:centos8 -``` -If you want to limit the supported/installed Python version(s), you can set the environment variable `PYTHON_TARGETS` before starting the bootstrap script. By only including a Python 3 version, you can prevent Python 2 from being installed, e.g.: -``` -env PYTHON_TARGETS="python3_8" CUSTOM_SNAPSHOT_URL="http://cvmfs-s0.eessi-hpc.org/snapshots" CUSTOM_SNAPSHOT_VERSION="20201126" singularity run docker://ghcr.io/eessi/bootstrap-prefix:centos8 -``` - -After starting the bootstrap have a long coffee... - -Once the bootstrap is completed, run the script to replace some paths with symlinks into the host OS: - -``` -scripts/prefix-symlink-host-paths.sh -``` - -In order to generate the right locales for your installation, see [this item](https://wiki.gentoo.org/wiki/Project:Prefix/FAQ#Add_an_en_US.UTF-8_locale) -of the Gentoo Prefix FAQ. - -### Adding the EESSI overlay and packages -Additional packages are added in the EESSI overlay, which is based on ComputeCanada. -You can add them manually or in an automated way by using Ansible, being Ansible the preferred way. Below you can find the two options explained. - -#### Ansible playbook (Option 1) -The installation of the EESSI-specific parts can be automatically executed by running the Ansible playbook `install.yml` inside the folder `ansible/playbooks`. -This playbook will install the [EESSI Gentoo overlay](https://github.com/EESSI/gentoo-overlay) and a set of packages, including `Lmod` and `archspec`. See the `README` in the `ansible/playbooks` folder for more details. - -#### Manually (Option 2) -First, set `EPREFIX` to the path containing your Gentoo Prefix installation, and start the prefix: -``` -export EPREFIX=/path/to/your/prefix -${EPREFIX}/startprefix -``` -Ensure that the configuration directory for repositories exists: -``` -mkdir -p ${EPREFIX}/etc/portage/repos.conf -``` -If you used `${PYTHON_TARGETS}` during the bootstrap, be sure to set it to the same value now, e.g.: -``` -export PYTHON_TARGETS="python3_7" -``` - -Next, configure and sync the overlay: -``` -emerge eselect-repository -eselect repository add eessi git https://github.com/EESSI/gentoo-overlay.git -emerge --sync eessi -``` - -After synchronizing the overlay, add the EESSI package set(s) that you would like to install, e.g. for set `2020.08`: -``` -mkdir ${EPREFIX}/etc/portage/sets/ -ln -s ${EPREFIX}/var/db/repos/eessi/etc/portage/sets/2020.08 ${EPREFIX}/etc/portage/sets/ -``` - -Finally, install the package set(s) defined at `${EPREFIX}/etc/portage/sets/`, e.g.: -``` -emerge @2020.08 -``` - -### Updating the Prefix -#### Packages -Updating packages can be as easy as -``` -emerge --sync -emerge -``` -If you run into problems, usually a newer ebuild is not suited to build in a prefix environment. -Try to mask latest versions: - -Create a mask file if not existing and mask newer versions from thin provisioning tools greater or equal to 0.7.6: -``` -echo ">=sys-block/thin-provisioning-tools-0.7.6" >> ${EPREFIX}/etc/portage/package.mask -``` - -#### Portage -Updating Portage requires the kernel source which corresponds to your running kernel on the host. Emerge will detect it in `/usr/src/linux`. - -Check your running kernel version with: -``` -cat /proc/version -Linux version 4.20.0-1.el7.elrepo.x86_64 (mockbuild@Build64R7) -``` - -On a Centos 7 host kernel sources are installed in `/usr/src/kernels`. Link `/usr/src/linux` to the appropiate kernel source after installation. Example for an `elrepo` kernel: -``` -rpm -ivh kernel-ml-devel-4.20.0-1.el7.elrepo.x86_64.rpm -cd /usr/src ; ln -s kernels/4.20.0-1.el7.elrepo.x86_64 linux -``` - - When ready update Portage from the Prefix environment: -``` -startprefix -emerge --oneshot sys-apps/portage -``` +The installation of the compatibility layer is implemented with an Ansible playbook and Ansible role, +which you can find in the `ansible` subdirectory. It will do the Gentoo Prefix bootstrap (which usually takes several hours to complete!), +add a lot of EESSI configurations and customizations, install a bunch of packages that we require for EESSI, and finally it will +run a ReFrame test suite to check the installation. + +To make the installation even easier, we provide a script `install_compatibility_layer.sh` that can be used on basically any host that has Apptainer installed, +without requiring special privileges. +The script will execute the Ansible playbook inside an Apptainer build container, ensuring that all dependencies (including Ansible itself) are available. +In order to be able to write to `/cvmfs`, the container will bind mount a directory from the host as `/cvmfs` inside the container. # License diff --git a/ansible/playbooks/README.md b/ansible/playbooks/README.md index daadfd81..73ddbb8d 100644 --- a/ansible/playbooks/README.md +++ b/ansible/playbooks/README.md @@ -11,10 +11,10 @@ all functionality for installing the EESSI compatibility layer. It performs the - add a given overlay to the installation; - use the Portage configuration files from that overlay, if applicable, by making symlinks to them; - install a given list of package sets; - - install a given list of additional packages. + - install a given list of additional packages; + - test the installation using ReFrame. The playbook `install.yml` will execute this role on a given server. -Note that if you want the role to install Gentoo Prefix, this particular task currently only supports Linux distributions based on RHEL 8 on the installation host. ## Configuration @@ -29,9 +29,6 @@ Before running the playbook, make sure the following settings are correct, and o ### CVMFS settings | Variable | Description | | --- | --- | -| cvmfs_start_transaction | Whether a CVMFS transaction should be start at the start | -| cvmfs_publish_transaction | Whether a CVMFS transaction should be published at the end | -| cvmfs_abort_transaction_on_failures | Whether a CVMFS transaction should be aborted on failures | | cvmfs_repository | Name of your CVMFS repository (used for the transaction) | ### Prefix and packages @@ -45,16 +42,17 @@ Before running the playbook, make sure the following settings are correct, and o | prefix_default_gcc | GCC compiler version to use as default compiler in Gentoo Prefix installation | | prefix_user_defined_trusted_dirs | List of paths to the user defined trusted dirs for glibc | | prefix_mask_packages | Contents of a [package.mask file](https://wiki.gentoo.org/wiki//etc/portage/package.mask) that should be used during the bootstrap | +| prefix_unmask_packages | Contents of a [package.unmask file](https://wiki.gentoo.org/wiki//etc/portage/package.unmask) that should be used during the bootstrap | | prefix_bootstrap_use_flags | Contents of [package.use file](https://wiki.gentoo.org/wiki//etc/portage/package.use) to put in place after bootstrap stage 3 | | prefix_use_builtin_bootstrap | Use the container's built-in bootstrap script? | | prefix_custom_bootstrap_script | Dictionary with the `local` source and `remote` destination of the bootstrap script | -| prefix_singularity_command | Singularity command for launching the container with the bootstrap script | -| prefix_source | Singularity container path used for the Prefix installtion | | prefix_source_options | Arguments to be passed to the Prefix bootstrap script | | prefix_install | Prefix installation command | | prefix_locales | List of locales to be generated | | package_sets | List of package sets to be installed | | prefix_packages | List of additional packages to be installed | +| prefix_remove_packages | List of packages to be removed after the bootstrap | +| reframe_venv | Path where a virtual environment will be created for the ReFrame installation | | symlinks_to_host | List of paths that should get a symlink to the corresponding host path | ### Logging diff --git a/ansible/playbooks/install.yml b/ansible/playbooks/install.yml index 75c1b123..82b23af2 100644 --- a/ansible/playbooks/install.yml +++ b/ansible/playbooks/install.yml @@ -1,7 +1,7 @@ # Playbook for installing the EESSI compatibility layer. --- - name: Install compatibility layer - hosts: cvmfsstratum0servers + hosts: localhost environment: PATH: "{{ gentoo_prefix_path }}/usr/sbin\ @@ -14,4 +14,3 @@ LC_ALL: C roles: - compatibility_layer - become: true diff --git a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml index d9e657a6..cef05bac 100644 --- a/ansible/playbooks/roles/compatibility_layer/defaults/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/defaults/main.yml @@ -1,6 +1,6 @@ # Defaults file for the compatibility layer role. --- -eessi_version: "2021.12" +eessi_version: "2023.03" custom_overlays: - name: eessi @@ -8,9 +8,6 @@ custom_overlays: url: https://github.com/EESSI/gentoo-overlay.git eclass-overrides: true -cvmfs_start_transaction: false -cvmfs_publish_transaction: false -cvmfs_abort_transaction_on_failures: false cvmfs_repository: pilot.eessi-hpc.org gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }} @@ -19,21 +16,18 @@ gentoo_prefix_path: /cvmfs/{{ cvmfs_repository }}/versions/{{ eessi_version }}/c gentoo_git_repo: https://github.com/gentoo/gentoo.git # Select a specific commit in the gentoo_git_repo that should be used for the bootstrap, # e.g. by checking: https://github.com/gentoo/gentoo/commits/master -gentoo_git_commit: 7eaa2512d1e6ddb44e3b41bbddf6c74723f234ce +gentoo_git_commit: 4ca74e7abe4f2b14e686267b517c59d43bb580b4 prefix_required_space: 15 GB -prefix_default_gcc: 9.4.0 +prefix_default_gcc: 9.5.0 prefix_user_defined_trusted_dirs: - "/cvmfs/{{ cvmfs_repository }}/host_injections/{{ eessi_version }}/compat/{{ eessi_host_os }}/{{ eessi_host_arch }}/lib" prefix_mask_packages: | - # avoid glibc 2.34, as it's causing issues with the bootstrap, and it's not compatible with CUDA 11. - # see https://github.com/EESSI/compatibility-layer/issues/137 + https://bugs.gentoo.org/824482 - >=sys-libs/glibc-2.34 # stick to GCC 9.x; using a too recent compiler in the compat layer complicates stuff in the software layer, # see for example https://github.com/EESSI/software-layer/issues/151 >=sys-devel/gcc-10 - # avoid libgcrypt 1.9.4 due to compiler errros on ppc64le, - # see https://github.com/EESSI/compatibility-layer/issues/134 + https://bugs.gentoo.org/825722 - =dev-libs/libgcrypt-1.9.4 +prefix_unmask_packages: | + # unmask older GCC to make it installable + =sys-devel/gcc-9* prefix_bootstrap_use_flags: | # make sure that gold linker is installed with binutils sys-devel/binutils gold @@ -41,12 +35,8 @@ prefix_use_builtin_bootstrap: false prefix_custom_bootstrap_script: local: "{{ playbook_dir }}/../../bootstrap-prefix.sh" remote: /tmp/bootstrap-prefix.sh -prefix_singularity_command: "singularity exec -B {{ gentoo_prefix_path }}:{{ gentoo_prefix_path }}" -prefix_source: "docker://ghcr.io/eessi/bootstrap-prefix:centos8" prefix_source_options: "{{ gentoo_prefix_path }} noninteractive" prefix_install: >- - SINGULARITYENV_USE_CPU_CORES={{ ansible_processor_vcpus }} - {{ prefix_singularity_command }} {{ prefix_source }} {{ prefix_use_builtin_bootstrap | ternary('/usr/local/bin/bootstrap-prefix.sh', prefix_custom_bootstrap_script.remote) }} {{ prefix_source_options }} @@ -64,6 +54,12 @@ package_sets: prefix_packages: +prefix_remove_packages: + - dev-lang/go + - dev-lang/go-bootstrap + +reframe_venv_dir: /tmp/reframe_venv + # List of locations that should get a symlink $EPREFIX/$LOCATION -> $LOCATION. # This ensures that things like user/group ids are correct/looked up in the right way in the Prefix environment. symlinks_to_host: diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml index 23fc8ea5..b1ebf2b5 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/add_overlay.yml @@ -1,19 +1,5 @@ # Add a custom overlay to the Gentoo Prefix installation. --- -- name: Add portage user if missing - become: true - ansible.builtin.lineinfile: - path: /etc/passwd - regexp: portage - line: portage:x:250:250:portage:/var/tmp/portage:/bin/false - -- name: Add portage group if missing - become: true - ansible.builtin.lineinfile: - path: /etc/group - regexp: portage - line: portage::250:portage - - name: Install equery command (dependency for the portage module) ansible.builtin.command: cmd: emerge gentoolkit @@ -21,6 +7,7 @@ - name: Install eselect-repository community.general.portage: + verbose: true package: eselect-repository state: present diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml new file mode 100644 index 00000000..cc0a93bc --- /dev/null +++ b/ansible/playbooks/roles/compatibility_layer/tasks/cleanup.yml @@ -0,0 +1,7 @@ +# Clean up +--- +- name: Remove redundant packages + community.general.portage: + package: "{{ item }}" + state: absent + with_items: "{{ prefix_remove_packages }}" diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml index 0875c385..4d0cb40e 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_packages.yml @@ -5,7 +5,6 @@ package: "@{{ item }}" state: present with_items: "{{ package_sets }}" - become: false tags: - set @@ -14,18 +13,3 @@ package: "{{ item }}" state: present with_items: "{{ prefix_packages }}" - become: false - -- name: "Get the username running the deployment (not root)" - ansible.builtin.command: whoami - changed_when: false - become: false - register: username_on_host - -- name: "Fix permissions after installing as portage/root" - ansible.builtin.file: - owner: "{{ username_on_host.stdout }}" - group: "{{ username_on_host.stdout }}" - path: "{{ gentoo_prefix_path }}" - recurse: true - become: false diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml index afeb1934..c787c7e0 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/install_prefix.yml @@ -1,46 +1,11 @@ # Install Gentoo Prefix. --- -- name: Fail if host OS is not supported - ansible.builtin.fail: - msg: | - Error: the operating system of the installation host is {{ ansible_os_family }} {{ ansible_distribution_version }}. - The task for installing Gentoo Prefix currently only supports Linux distributions based on RHEL 8. - when: not (ansible_os_family == "RedHat" and ansible_distribution_major_version is version("8", "==")) - -- name: "Install EPEL" - ansible.builtin.yum: - name: - - https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm - disable_gpg_check: true - state: present - tags: - - build_prefix - -- name: "Install other requirements (Singularity and git)" - ansible.builtin.yum: - name: - - singularity - - git - state: present - tags: - - build_prefix - -- name: "Get the username running the deployment (not root)" - ansible.builtin.shell: set -o pipefail && who | grep -v root | awk '{print $1}' | uniq - become: false - register: username_on_host - changed_when: true - tags: - - build_prefix - - name: "Create Gentoo prefix path and log directory" ansible.builtin.file: path: "{{ item }}" state: directory recurse: true - owner: "{{ username_on_host.stdout }}" - group: "{{ username_on_host.stdout }}" mode: "0755" with_items: - "{{ gentoo_prefix_path }}" @@ -76,7 +41,6 @@ path: "{{ gentoo_prefix_path }}/etc/portage/repos.conf" state: directory mode: 0755 - when: prefix_mask_packages is defined and prefix_mask_packages | length > 0 tags: - build_prefix @@ -171,6 +135,51 @@ tags: - build_prefix +- name: "Fix missing profile for arm64 - create directory" + ansible.builtin.file: + path: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64" + mode: '0755' + state: directory + when: eessi_host_arch == "aarch64" + tags: + - build_prefix + +- name: "Fix missing profile for arm64 - eapi" + ansible.builtin.copy: + content: "5" + dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64/eapi" + mode: 0644 + when: eessi_host_arch == "aarch64" + tags: + - build_prefix + +- name: "Fix missing profile for arm64 - parent" + ansible.builtin.copy: + content: | + ../../../default/linux/arm64/17.0 + .. + dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64/parent" + mode: 0644 + when: eessi_host_arch == "aarch64" + tags: + - build_prefix + +- name: "Fix missing profile for arm64 - make.defaults" + ansible.builtin.copy: + content: | + # Copyright 1999-2018 Gentoo Foundation + # Distributed under the terms of the GNU General Public License v2 + + ACCEPT_KEYWORDS="arm64 ~arm64" + + # Not sure if this is required as well, seems to be the default: + # CHOST=aarch64-unknown-linux-gnu + dest: "{{ gentoo_prefix_path }}/var/db/repos/gentoo/profiles/prefix/linux/arm64/make.defaults" + mode: 0644 + when: eessi_host_arch == "aarch64" + tags: + - build_prefix + - name: "Mask packages for the bootstrap" ansible.builtin.copy: dest: "{{ gentoo_prefix_path }}/etc/portage/package.mask" @@ -180,17 +189,19 @@ tags: - build_prefix -- name: "Give build user recursive user and group ownership of {{ gentoo_prefix_path }}" - ansible.builtin.file: - dest: "{{ gentoo_prefix_path }}" - owner: "{{ username_on_host.stdout }}" - group: "{{ username_on_host.stdout }}" - recurse: true +- name: "Unmask packages for the bootstrap" + ansible.builtin.copy: + dest: "{{ gentoo_prefix_path }}/etc/portage/package.unmask" + content: "{{ prefix_unmask_packages }}" + mode: 0644 + when: prefix_unmask_packages is defined and prefix_unmask_packages | length > 0 tags: - build_prefix - name: "Run Gentoo Prefix bootstrap stages 1-3 via {{ prefix_install }}" ansible.builtin.shell: set -o pipefail && ( {{ prefix_install }} | tee -a {{ prefix_build_log }} | grep -E '^(>>> Installing|\\* )' ) + args: + executable: /bin/bash become: false changed_when: true environment: @@ -209,6 +220,8 @@ - name: "Continue Gentoo Prefix bootstrap via {{ prefix_install }}" ansible.builtin.shell: set -o pipefail && ( {{ prefix_install }} | tee -a {{ prefix_build_log }} | grep -E '^(>>> Installing|\\* )' ) + args: + executable: /bin/bash become: false changed_when: true tags: diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml index 2504c77c..0a93747e 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/main.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/main.yml @@ -1,6 +1,5 @@ # Main task which: # - checks the given path for a Prefix installation, and installs it if necessary; -# - starts (and publishes at the end) a CVMFS transaction, if requested; # - calls the tasks for adding the overlay and installation of sets and packages; # - does some fixes and other modifications in the Prefix installation (e.g. setting the locale). --- @@ -21,44 +20,25 @@ ansible.builtin.include_tasks: install_prefix.yml when: not startprefix.stat.exists -- name: Start transaction - ansible.builtin.command: "cvmfs_server transaction {{ cvmfs_repository }}" - when: cvmfs_start_transaction +- name: Configure the Prefix + ansible.builtin.include_tasks: prefix_configuration.yml -- name: Make customizations to our Prefix installation - block: - - name: Configure the Prefix - ansible.builtin.include_tasks: prefix_configuration.yml +- name: Add EESSI overlay + ansible.builtin.include_tasks: add_overlay.yml - - name: Add EESSI overlay - ansible.builtin.include_tasks: add_overlay.yml - args: - apply: - become: false +- name: Set the glibc trusted dirs + ansible.builtin.include_tasks: set_glibc_trusted_dirs.yml - - name: Set the glibc trusted dirs - ansible.builtin.include_tasks: set_glibc_trusted_dirs.yml +- name: Install additional packages + ansible.builtin.include_tasks: install_packages.yml - - name: Install additional packages - ansible.builtin.include_tasks: install_packages.yml +- name: Create symlinks to host files + ansible.builtin.include_tasks: create_host_symlinks.yml - - name: Create symlinks to host files - ansible.builtin.include_tasks: create_host_symlinks.yml +- name: Clean up + ansible.builtin.include_tasks: cleanup.yml - - name: Test the Prefix installation - ansible.builtin.include_tasks: test.yml - tags: - - test - - - name: Publish transaction - ansible.builtin.command: "cvmfs_server publish {{ cvmfs_repository }}" - when: cvmfs_start_transaction and cvmfs_publish_transaction - - rescue: - - name: Abort transaction - ansible.builtin.command: "cvmfs_server abort {{ cvmfs_repository }}" - when: cvmfs_start_transaction and cvmfs_abort_transaction_on_failures - - - name: Exit because of failure - ansible.builtin.fail: - msg: "Task {{ ansible_failed_task }} failed, with result {{ ansible_failed_result }}." +- name: Test the Prefix installation + ansible.builtin.include_tasks: test.yml + tags: + - test diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml index 997a2c79..80a5076f 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/set_glibc_trusted_dirs.yml @@ -12,7 +12,6 @@ package: sys-libs/glibc noreplace: false oneshot: true - become: false environment: EXTRA_EMAKE: "user-defined-trusted-dirs={{ prefix_user_defined_trusted_dirs | join(':') }}" when: diff --git a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml index 181fb0ed..631ad3e5 100644 --- a/ansible/playbooks/roles/compatibility_layer/tasks/test.yml +++ b/ansible/playbooks/roles/compatibility_layer/tasks/test.yml @@ -2,16 +2,32 @@ # of the compatibility layer installation. --- +- name: Check if ReFrame is installed + ansible.builtin.command: reframe --version >/dev/null 2>&1 + register: reframe_exists + ignore_errors: true + changed_when: false + tags: + - test + +- name: Install Reframe using pip if it's not installed yet + ansible.builtin.pip: + name: ReFrame-HPC + virtualenv: "{{ reframe_venv_dir }}" + virtualenv_command: python3 -m venv + when: reframe_exists.rc != 0 + - name: Copy ReFrame test file ansible.builtin.copy: src: "{{ playbook_dir }}/../../test/compat_layer.py" - dest: /tmp/compat_layer.py + dest: "{{ reframe_venv_dir }}/compat_layer.py" mode: 0644 tags: - test - name: Run ReFrame tests - ansible.builtin.command: reframe -r -v -c /tmp/compat_layer.py + ansible.builtin.command: + cmd: "{{ reframe_venv_dir + '/bin/' if reframe_exists.rc != 0 else '' }}reframe -r -v -c {{ reframe_venv_dir }}/compat_layer.py" environment: EESSI_VERSION: "{{ eessi_version }}" EESSI_OS: "{{ eessi_host_os }}" diff --git a/bootstrap-prefix.sh b/bootstrap-prefix.sh index b45b7f85..c812abbf 100755 --- a/bootstrap-prefix.sh +++ b/bootstrap-prefix.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Copyright 2006-2021 Gentoo Authors; Distributed under the GPL v2 +# Copyright 2006-2022 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 -trap 'exit 1' TERM KILL INT QUIT ABRT +trap 'exit 1' TERM INT QUIT ABRT # RAP (libc) mode is triggered on Linux kernel and glibc. is-rap() { [[ ${PREFIX_DISABLE_RAP} != "yes" && ${CHOST} = *linux-gnu* ]]; } @@ -39,16 +40,25 @@ emake() { [[ $* == *install* ]] \ && estatus "stage1: installing ${PWD##*/}" \ || estatus "stage1: building ${PWD##*/}" - v $MAKE ${MAKEOPTS} "$@" || return 1 + v ${MAKE} ${MAKEOPTS} "$@" || return 1 } efetch() { if [[ ! -e ${DISTDIR}/${1##*/} ]] ; then - if [[ ${OFFLINE_MODE} ]]; then - echo "I need ${1##*/} from $1 in $DISTDIR, can you give it to me?" - read - [[ -e ${DISTDIR}/${1##*/} ]] && return 0 - # Give fetch a try + mkdir -p "${DISTDIR}" >& /dev/null + + # Try fetching from local mirrors first, as this requires no connection + for loc in ${GENTOO_MIRRORS} ; do + if [[ ${loc} = /* && -e "${loc}/${1##*/}" ]]; then + cp "${loc}/${1##*/}" "${DISTDIR}/${1##*/}" && return 0 + fi + done + + if [[ ${OFFLINE_MODE} ]] ; then + echo "I need ${1##*/} from $1 in $DISTDIR, can you give it to me?" + read + [[ -e ${DISTDIR}/${1##*/} ]] && return 0 + # Give fetch a try fi if [[ -z ${FETCH_COMMAND} ]] ; then @@ -74,12 +84,11 @@ efetch() { fi fi - mkdir -p "${DISTDIR}" >& /dev/null einfo "Fetching ${1##*/}" estatus "stage1: fetching ${1##*/}" pushd "${DISTDIR}" > /dev/null - # try for mirrors first, fall back to distfiles, then try given location + # Try for mirrors first, fall back to distfiles, then try given location local locs=( ) local loc for loc in ${GENTOO_MIRRORS} ${DISTFILES_G_O} ${DISTFILES_PFX}; do @@ -160,7 +169,7 @@ configure_toolchain() { *-darwin*) # handled below ;; - *-freebsd*) + *-freebsd* | *-openbsd*) # comes with clang, handled below ;; *) @@ -190,9 +199,11 @@ configure_toolchain() { einfo "Triggering Darwin with GCC toolchain" compiler_stage1+=" sys-apps/darwin-miscutils" local ccvers="$(unset CHOST; /usr/bin/gcc --version 2>/dev/null)" + local isgcc= case "${ccvers}" in *"(GCC) 4.2.1 "*) linker="=sys-devel/binutils-apple-3.2.6" + isgcc=true ;; *"(GCC) 4.0.1 "*) linker="=sys-devel/binutils-apple-3.2.6" @@ -200,6 +211,7 @@ configure_toolchain() { compiler_stage1+=" sys-devel/gcc-apple =sys-devel/binutils-apple-3.2.6" + isgcc=true ;; *"Apple clang version "*|*"Apple LLVM version "*) # recent binutils-apple are hard to build (C++11 @@ -214,7 +226,15 @@ configure_toolchain() { return 1 ;; esac - compiler_stage1+=" sys-devel/gcc" + if [[ ${isgcc} == true ]] ; then + # current compiler (gcc-11) requires C++11, which is + # available since 4.8, so need to bootstrap with <11 + compiler_stage1+=" 2*256^3 + 6*256^2 + 32 * 256 + 1 = 33955841 - kver() { uname -r|cut -d\- -f1|awk -F. '{for (i=1; i<=NF; i++){s+=lshift($i,(4-i)*8)};print s}'; } - # >=glibc-2.20 requires >=linux-2.6.32. - profile-kernel() { - if [[ $(kver) -ge 50462720 ]] ; then # 3.2 - echo kernel-3.2+ - elif [[ $(kver) -ge 33955840 ]] ; then # 2.6.32 - echo kernel-2.6.32+ - elif [[ $(kver) -ge 33951744 ]] ; then # 2.6.16 - echo kernel-2.6.16+ - elif [[ $(kver) -ge 33947648 ]] ; then # 2.6 - echo kernel-2.6+ - fi - } + einfo "Setting up some guessed defaults" local FS_INSENSITIVE=0 touch "${ROOT}"/FOO.$$ [[ -e ${ROOT}/foo.$$ ]] && FS_INSENSITIVE=1 rm "${ROOT}"/FOO.$$ - if [[ ! -f ${ROOT}/etc/portage/make.conf ]] ; then + [[ ! -e "${MAKE_CONF_DIR}" ]] && mkdir -p -- "${MAKE_CONF_DIR}" + if [[ ! -f ${MAKE_CONF_DIR}/0100_bootstrap_prefix_make.conf ]] ; then { echo "# Added by bootstrap-prefix.sh for ${CHOST}" echo 'USE="unicode nls"' @@ -340,7 +345,7 @@ bootstrap_setup() { echo "USE=\"\${USE} ${MAKE_CONF_ADDITIONAL_USE}\"" [[ ${OFFLINE_MODE} ]] && \ echo 'FETCHCOMMAND="bash -c \"echo I need \${FILE} from \${URI} in \${DISTDIR}; read\""' - } > "${ROOT}"/etc/portage/make.conf + } > "${MAKE_CONF_DIR}/0100_bootstrap_prefix_make.conf" fi if is-rap ; then @@ -364,6 +369,30 @@ bootstrap_setup() { fi [[ -f ${ROOT}/etc/resolv.conf ]] || ln -s {,"${ROOT}"}/etc/resolv.conf [[ -f ${ROOT}/etc/hosts ]] || cp {,"${ROOT}"}/etc/hosts + fi + + bootstrap_profile +} + +bootstrap_profile() { + local profile="" + + # 2.6.32.1 -> 2*256^3 + 6*256^2 + 32 * 256 + 1 = 33955841 + kver() { uname -r|cut -d\- -f1|awk -F. '{for (i=1; i<=NF; i++){s+=lshift($i,(4-i)*8)};print s}'; } + # >=glibc-2.20 requires >=linux-2.6.32. + profile-kernel() { + if [[ $(kver) -ge 50462720 ]] ; then # 3.2 + echo kernel-3.2+ + elif [[ $(kver) -ge 33955840 ]] ; then # 2.6.32 + echo kernel-2.6.32+ + elif [[ $(kver) -ge 33951744 ]] ; then # 2.6.16 + echo kernel-2.6.16+ + elif [[ $(kver) -ge 33947648 ]] ; then # 2.6 + echo kernel-2.6+ + fi + } + + if is-rap ; then local profile_linux=default/linux/ARCH/17.0/prefix/$(profile-kernel) else local profile_linux=prefix/linux/ARCH @@ -382,34 +411,25 @@ bootstrap_setup() { rev=${CHOST##*darwin} profile="prefix/darwin/macos/10.$((rev - 4))/x64" ;; - x86_64-apple-darwin20) + *64-apple-darwin2[0123456789]) # Big Sur is 11.0 - rev=${CHOST##*darwin} - profile="prefix/darwin/macos/11.$((rev - 20))/x64" - ;; - x86_64-apple-darwin2[123456789]) # Monterey is 12.0 + # Ventura is 13.0 rev=${CHOST##*darwin} - profile="prefix/darwin/macos/12.$((rev - 21))/x64" - ;; - arm64-apple-darwin20) - rev=${CHOST##*darwin} - profile="prefix/darwin/macos/11.$((rev - 20))/arm64" - ;; - # TODO: Come up with something better for macOS 11+ - x86_64-apple-darwin2[123456789]) - # Monterey is 12.0 - rev=${CHOST##*darwin} - profile="prefix/darwin/macos/12.$((rev - 21))/x64" - ;; - arm64-apple-darwin2[123456789]) - # Monterey is 12.0 - rev=${CHOST##*darwin} - profile="prefix/darwin/macos/12.$((rev - 21))/arm64" + case ${CHOST%%-*} in + x86_64) arch=x64 ;; + arm64) arch=arm64 ;; + *) arch=error ;; + esac + profile="prefix/darwin/macos/$((rev - 9)).0/${arch}" ;; i*86-pc-linux-gnu) profile=${profile_linux/ARCH/x86} ;; + riscv64-pc-linux-gnu) + profile=${profile_linux/ARCH/riscv} + profile=${profile/17.0/20.0/rv64gc/lp64d} + ;; x86_64-pc-linux-gnu) profile=${profile_linux/ARCH/amd64} profile=${profile/17.0/17.1/no-multilib} @@ -423,6 +443,10 @@ bootstrap_setup() { powerpc64le-unknown-linux-gnu) profile=${profile_linux/ARCH/ppc64le} ;; + riscv-pc-unknown-linux-gnu) + profile=${profile_linux/ARCH/riscv} + profile=${profile/17.0/20.0/rv64gc/lp64d} + ;; aarch64-unknown-linux-gnu) profile=${profile_linux/ARCH/arm64} ;; @@ -482,21 +506,10 @@ bootstrap_setup() { einfo "Your profile is set to ${fullprofile}." fi - is-rap && cat >> "${ROOT}"/etc/portage/make.profile/make.defaults <<-'EOF' - # For baselayout-prefix in stage2 only. - ACCEPT_KEYWORDS="~${ARCH}-linux" - EOF - - # bug #788613 avoid gcc-11 during stage 2/3 prior sync/emerge -e - is-rap && cat >> "${ROOT}"/etc/portage/make.profile/package.mask <<-EOF - # during bootstrap mask, bug #788613 - >=sys-devel/gcc-11 - EOF - # Use package.use to disable in the portage tree to be shared between # stage2 and stage3. The hack will be undone during tree sync in stage3. cat >> "${ROOT}"/etc/portage/make.profile/package.use <<-EOF - # disable bootstrapping libcxx* with libunwind + # Disable bootstrapping libcxx* with libunwind sys-libs/libcxxabi -libunwind sys-libs/libcxx -libunwind # Most binary Linux distributions seem to fancy toolchains that @@ -513,11 +526,6 @@ bootstrap_setup() { sys-devel/native-cctools EOF - [[ ${CHOST} == arm64-*-darwin* ]] && - cat >> "${ROOT}"/etc/portage/package.accept_keywords <<-EOF - =sys-devel/gcc-11_pre20200206 ** - EOF - # Strange enough, -cxx causes wrong libtool config on Cygwin, # but we require a C++ compiler there anyway - so just use it. [[ ${CHOST} == *-cygwin* ]] || @@ -565,11 +573,10 @@ do_tree() { else efetch "$1/$2" || return 1 fi - [[ -e ${PORTDIR} ]] || mkdir -p ${PORTDIR} einfo "Unpacking, this may take a while" estatus "stage1: unpacking Portage tree" - bzip2 -dc ${DISTDIR}/$2 | \ - tar -xf - -C ${PORTDIR} --strip-components=1 || return 1 + bzip2 -dc ${DISTDIR}/$2 | tar -xf - -C ${PORTDIR} --strip-components=1 + [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 touch ${PORTDIR}/.unpacked fi } @@ -577,7 +584,7 @@ do_tree() { bootstrap_tree() { # RAP uses the latest gentoo main repo snapshot to bootstrap. is-rap && LATEST_TREE_YES=1 - local PV="20211105" + local PV="20221104" if [[ -n ${LATEST_TREE_YES} ]]; then do_tree "${SNAPSHOT_URL}" portage-latest.tar.bz2 else @@ -596,7 +603,7 @@ bootstrap_tree() { } bootstrap_startscript() { - local theshell=$(echo "${SHELL##*/}" | cut -f1 -d' ') + local theshell=${SHELL##*/} if [[ ${theshell} == "sh" ]] ; then einfo "sh is a generic shell, using bash instead" theshell="bash" @@ -644,8 +651,8 @@ bootstrap_portage() { # STABLE_PV that is known to work. Intended for power users only. ## It is critical that STABLE_PV is the lastest (non-masked) version that is ## included in the snapshot for bootstrap_tree. - STABLE_PV="3.0.21" - [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.21" + STABLE_PV="3.0.30.1" + [[ ${TESTING_PV} == latest ]] && TESTING_PV="3.0.30.1" PV="${TESTING_PV:-${STABLE_PV}}" A=prefix-portage-${PV}.tar.bz2 einfo "Bootstrapping ${A%.tar.*}" @@ -658,7 +665,8 @@ bootstrap_portage() { rm -rf "${S}" >& /dev/null mkdir -p "${S}" >& /dev/null cd "${S}" - bzip2 -dc "${DISTDIR}/${A}" | tar -xf - || return 1 + bzip2 -dc "${DISTDIR}/${A}" | tar -xf - + [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}/prefix-portage-${PV}" cd "${S}" @@ -707,7 +715,7 @@ bootstrap_portage() { local tmpportdir=${ROOT}/tmp/${PORTDIR#${ROOT}} [[ -e "${tmpportdir}" ]] || ln -s "${PORTDIR}" "${tmpportdir}" - for d in "${ROOT}"/tmp/usr/lib/python?.?; do + for d in "${ROOT}"/tmp/usr/lib/python$(python_ver); do [[ -e ${d}/portage ]] || ln -s "${ROOT}"/tmp/usr/lib/portage/lib/portage ${d}/portage [[ -e ${d}/_emerge ]] || ln -s "${ROOT}"/tmp/usr/lib/portage/lib/_emerge ${d}/_emerge done @@ -753,7 +761,8 @@ bootstrap_simple() { bz2) decomp=bzip2 ;; gz|"") decomp=gzip ;; esac - ${decomp} -dc "${DISTDIR}"/${A} | tar -xf - || return 1 + ${decomp} -dc "${DISTDIR}"/${A} | tar -xf - + [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}"/${PN}-${PV} cd "${S}" @@ -808,18 +817,21 @@ bootstrap_gnu() { rm -rf "${S}" mkdir -p "${S}" cd "${S}" - if [[ ${t} == "tar.gz" ]] ; then - gzip -dc "${DISTDIR}"/${URL##*/} | tar -xf - || continue - elif [[ ${t} == "tar.xz" ]] ; then - xz -dc "${DISTDIR}"/${URL##*/} | tar -xf - || continue - elif [[ ${t} == "tar.bz2" ]] ; then - bzip2 -dc "${DISTDIR}"/${URL##*/} | tar -xf - || continue - elif [[ ${t} == "tar" ]] ; then - tar -xf "${DISTDIR}"/${A} || continue - else - einfo "unhandled extension: $t" - return 1 - fi + case ${t} in + tar.xz) decomp=xz ;; + tar.bz2) decomp=bzip2 ;; + tar.gz) decomp=gzip ;; + tar) + tar -xf "${DISTDIR}"/${A} || continue + break + ;; + *) + einfo "unhandled extension: $t" + return 1 + ;; + esac + ${decomp} -dc "${DISTDIR}"/${URL##*/} | tar -xf - + [[ ${PIPESTATUS[*]} == '0 0' ]] || continue break done S="${S}"/${PN}-${PV} @@ -874,13 +886,15 @@ bootstrap_gnu() { # doesn't match sed -i -e '/_GL_WARN_ON_USE (gets/d' lib/stdio.in.h lib/stdio.h - # macOS 10.13 have an issue with %n, which crashes m4 - efetch "http://rsync.prefix.bitzolder.nl/sys-devel/m4/files/m4-1.4.18-darwin17-printf-n.patch" || return 1 - patch -p1 < "${DISTDIR}"/m4-1.4.18-darwin17-printf-n.patch || return 1 + if [[ ${PV} == "1.4.18" ]] ; then + # macOS 10.13 have an issue with %n, which crashes m4 + efetch "http://rsync.prefix.bitzolder.nl/sys-devel/m4/files/m4-1.4.18-darwin17-printf-n.patch" || return 1 + patch -p1 < "${DISTDIR}"/m4-1.4.18-darwin17-printf-n.patch || return 1 - # Bug 715880 - efetch http://dev.gentoo.org/~heroxbd/m4-1.4.18-glibc228.patch || return 1 - patch -p1 < "${DISTDIR}"/m4-1.4.18-glibc228.patch || return 1 + # Bug 715880 + efetch http://dev.gentoo.org/~heroxbd/m4-1.4.18-glibc228.patch || return 1 + patch -p1 < "${DISTDIR}"/m4-1.4.18-glibc228.patch || return 1 + fi fi fix_config_sub @@ -898,17 +912,21 @@ bootstrap_gnu() { export ac_cv_path_POD2MAN=no # Darwin9 in particular doesn't compile when using system readline, - # but we don't need any groovy input at all, so just disable it, - # except for Cygwin, where the patch above would fail to compile - [[ ${PN} == "bash" && ${CHOST} != *-cygwin* ]] \ - && myconf="${myconf} --disable-readline" + # but we don't need any groovy input handling at all, so just disable it + [[ ${PN} == "bash" ]] && myconf="${myconf} --disable-readline" + + # On e.g. musl systems bash will crash with a malloc error if we use + # bash' internal malloc, so disable it during it this stage + [[ ${PN} == "bash" ]] && \ + myconf="${myconf} --without-bash-malloc" - # ensure we don't read system-wide shell initialisation, it may + # Ensure we don't read system-wide shell initialisation, it may # contain cruft, bug #650284 [[ ${PN} == "bash" ]] && \ export CPPFLAGS="${CPPFLAGS} \ - -DSYS_BASHRC=\'\"${ROOT}/etc/bash/bashrc\"\' \ - -DSYS_BASH_LOGOUT=\'\"${ROOT}/etc/bash/bash_logout\"\'" + -DSYS_BASHRC=\\\"${ROOT}/etc/bash/bashrc\\\" \ + -DSYS_BASH_LOGOUT=\\\"${ROOT}/etc/bash/bash_logout\\\" \ + " # Don't do ACL stuff on Darwin, especially Darwin9 will make # coreutils completely useless (install failing on everything) @@ -979,15 +997,24 @@ bootstrap_gnu() { einfo "${A%.tar.*} successfully bootstrapped" } -PYTHONMAJMIN=3.9 # keep this number in line with PV below for stage1,2 +python_ver() { + if [[ ${CHOST} == *-cygwin* ]] ; then + echo 3.9 # keep this number in line with PV below for stage1,2 + else + echo 3.10 # keep this number in line with PV below for stage1,2 + fi +} + bootstrap_python() { - PV=3.9.6 + if [[ ${CHOST} == *-cygwin* ]] ; then + PV=3.9.9 + else + PV=3.10.4 + fi A=Python-${PV}.tar.xz einfo "Bootstrapping ${A%.tar.*}" - # don't really want to put this on the mirror, since they are - # non-vanilla sources, bit specific for us - efetch ${DISTFILES_URL}/${A} || return 1 + efetch https://www.python.org/ftp/python/${PV}/${A} einfo "Unpacking ${A%.tar.*}" export S="${PORTAGE_TMPDIR}/python-${PV}" @@ -1007,24 +1034,32 @@ bootstrap_python() { case ${CHOST} in (*-*-cygwin*) - # apply patches from cygwinports much like the ebuild does - local gitrev pf pn - gitrev="71f2ac2444946c97d892be3892e47d2a509e0e96" # python36 3.6.8 - efetch "https://github.com/cygwinports/python36/archive/${gitrev}.tar.gz" \ + local cygpyver pf pn patch_folder + + # ideally the version of python used by bootstrap would be one + # that cygwin has packaged if we don't do exact matches on the + # version then some patches may not apply cleanly + + cygpyver="3.9.9-1" + efetch "https://mirrors.kernel.org/sourceware/cygwin/x86_64/release/python39/python39-${cygpyver}-src.tar.xz" \ || return 1 - gzip -dc "${DISTDIR}"/${gitrev}.tar.gz | tar -xf - + xz -dc "${DISTDIR}"/"python39-${cygpyver}-src.tar.xz" | tar -xf - [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 + patch_folder="python39-${cygpyver}.src" + for pf in $( sed -ne '/PATCH_URI="/,/"/{s/.*="//;s/".*$//;p}' \ - < python36-${gitrev}/python3.cygport + < "${patch_folder}/python39.cygport" \ + | grep -v rpm-wheels | grep -v revert-bpo ); do - pf="python36-${gitrev}/${pf}" + pf="${patch_folder}/${pf}" for pn in {1..2} fail; do if [[ ${pn} == fail ]]; then eerror "failed to apply ${pf}" return 1 fi - patch -N -p${pn} -i "${pf}" --dry-run >/dev/null 2>&1 || continue + patch -N -p${pn} -i "${pf}" --dry-run >/dev/null 2>&1 \ + || continue echo "applying (-p${pn}) ${pf}" patch -N -p${pn} -i "${pf}" || return 1 break @@ -1050,10 +1085,11 @@ bootstrap_python() { efetch "https://dev.gentoo.org/~sam/distfiles/dev-lang/python/python-3.9.6-darwin9_pthreadid.patch" patch -p1 < "${DISTDIR}"/python-3.9.6-darwin9_pthreadid.patch ;; - (arm64-*-darwin*) - # Teach Python a new trick (arm64) + (*-openbsd*) + # OpenBSD is not a multilib system sed -i \ - -e "/Unexpected output of 'arch' on OSX/d" \ + -e '0,/#if defined(__ANDROID__)/{s/ANDROID/OpenBSD/}' \ + -e '0,/MULTIARCH=/{s/\(MULTIARCH\)=.*/\1=""/}' \ configure ;; esac @@ -1070,7 +1106,7 @@ bootstrap_python() { local myconf="" - case $CHOST in + case ${CHOST} in (x86_64-*-*|sparcv9-*-*) export CFLAGS="-m64" ;; @@ -1079,7 +1115,7 @@ bootstrap_python() { ;; esac - case $CHOST in + case ${CHOST} in *-*-cygwin*) # --disable-shared would link modules against "python.exe" # so renaming to "pythonX.Y.exe" will break them. @@ -1106,12 +1142,16 @@ bootstrap_python() { export CPPFLAGS="-I${ROOT}/tmp/usr/include" export LDFLAGS="${CFLAGS} -L${ROOT}/tmp/usr/lib" # set correct flags for runtime for ELF platforms - case $CHOST in + case ${CHOST} in *-linux*) # GNU ld LDFLAGS="${LDFLAGS} -Wl,-rpath,${ROOT}/tmp/usr/lib ${libdir}" LDFLAGS="${LDFLAGS} -Wl,-rpath,${libdir#-L}" ;; + *-openbsd*) + # LLD + LDFLAGS="${LDFLAGS} -Wl,-rpath,${ROOT}/tmp/usr/lib" + ;; *-solaris*) # Sun ld LDFLAGS="${LDFLAGS} -R${ROOT}/tmp/usr/lib ${libdir}" @@ -1128,9 +1168,9 @@ bootstrap_python() { einfo "Compiling ${A%.tar.*}" - # some ancient versions of hg fail with "hg id -i", so help - # configure to not find them using HAS_HG - # do not find libffi via pkg-config using PKG_CONFIG + # - Some ancient versions of hg fail with "hg id -i", so help + # configure to not find them using HAS_HG (TODO: obsolete?) + # - Do not find libffi via pkg-config using PKG_CONFIG HAS_HG=no \ PKG_CONFIG= \ econf \ @@ -1168,7 +1208,8 @@ bootstrap_cmake_core() { rm -rf "${S}" mkdir -p "${S}" cd "${S}" - gzip -dc "${DISTDIR}"/${A} | tar -xf - || return 1 + gzip -dc "${DISTDIR}"/${A} | tar -xf - + [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}"/cmake-${PV} cd "${S}" @@ -1206,7 +1247,7 @@ bootstrap_cmake() { } bootstrap_zlib_core() { - # use 1.2.8 by default, current bootstrap guides + # Use 1.2.8 by default, current bootstrap guides PV="${1:-1.2.8}" A=zlib-${PV}.tar.gz @@ -1219,11 +1260,12 @@ bootstrap_zlib_core() { rm -rf "${S}" mkdir -p "${S}" cd "${S}" - if [[ ${A} == *.tar.gz ]] ; then - gzip -dc "${DISTDIR}"/${A} | tar -xf - || return 1 - else - bzip2 -dc "${DISTDIR}"/${A} | tar -xf - || return 1 - fi + case ${A} in + *.tar.gz) decomp=gzip ;; + *) decomp=bzip2 ;; + esac + ${decomp} -dc "${DISTDIR}"/${A} | tar -xf - + [[ ${PIPESTATUS[*]} == '0 0' ]] || return 1 S="${S}"/zlib-${PV} cd "${S}" @@ -1321,7 +1363,7 @@ bootstrap_coreutils() { # 8.16 is the last version released as tar.gz # 8.18 is necessary for macOS High Sierra (darwin17) and converted # to tar.gz for this case - bootstrap_gnu coreutils 8.30 || \ + bootstrap_gnu coreutils 8.32 || bootstrap_gnu coreutils 8.30 || \ bootstrap_gnu coreutils 8.16 || bootstrap_gnu coreutils 8.17 } @@ -1372,7 +1414,7 @@ bootstrap_bison() { } bootstrap_m4() { - bootstrap_gnu m4 1.4.18 # version is patched, so beware + bootstrap_gnu m4 1.4.19 || bootstrap_gnu m4 1.4.18 # version is patched, so beware } bootstrap_gzip() { @@ -1390,6 +1432,8 @@ bootstrap_bzip2() { } bootstrap_libressl() { + bootstrap_simple libressl 3.4.3 gz \ + https://ftp.openbsd.org/pub/OpenBSD/LibreSSL || \ bootstrap_simple libressl 3.2.4 gz \ https://ftp.openbsd.org/pub/OpenBSD/LibreSSL || \ bootstrap_simple libressl 2.8.3 gz \ @@ -1416,7 +1460,9 @@ bootstrap_stage_host_gentoo() { (bootstrap_tree) || return 1 # setup a profile - [[ -e ${ROOT}/etc/portage/make.profile && -e ${ROOT}/etc/portage/make.conf ]] || (bootstrap_setup) || return 1 + [[ -e ${ROOT}/etc/portage/make.profile && \ + -e ${MAKE_CONF_DIR}/0100_bootstrap_prefix_make.conf ]] \ + || (bootstrap_setup) || return 1 prepare_portage } @@ -1426,8 +1472,8 @@ bootstrap_stage1() { # bits-size of the compiler, which needs not to match what we're # bootstrapping for. This is no problem since they're just tools, # for which it really doesn't matter how they run, as long AS they - # run. For libraries, this is different, since they are relied on - # by packages we emerge lateron. + # run. For libraries, this is different, since they are relied upon + # by packages we emerge later on. # Changing this to compile the tools for the bits the bootstrap is # for, is a BAD idea, since we're extremely fragile here, so # whatever the native toolchain is here, is what in general works @@ -1444,10 +1490,12 @@ bootstrap_stage1() { configure_toolchain export CC CXX - # run all bootstrap_* commands in a subshell since the targets + # Run all bootstrap_* commands in a subshell since the targets # frequently pollute the environment using exports which affect # packages following (e.g. zlib builds 64-bits) + local CP + # don't rely on $MAKE, if make == gmake packages that call 'make' fail [[ -x ${ROOT}/tmp/usr/bin/make ]] \ || [[ $(make --version 2>&1) == *GNU" Make "4* ]] \ @@ -1471,9 +1519,20 @@ bootstrap_stage1() { || [[ $(bison --version 2>&1) == *GNU" "Bison") "2.[3-7]* ]] \ || [[ $(bison --version 2>&1) == *GNU" "Bison") "[3-9]* ]] \ || (bootstrap_bison) || return 1 - [[ -x ${ROOT}/tmp/usr/bin/uniq ]] \ - || [[ $(uniq --version 2>&1) == *"(GNU coreutils) "[6789]* ]] \ - || (bootstrap_coreutils) || return 1 + if [[ ! -x ${ROOT}/tmp/usr/bin/uniq ]]; then + # If the system has a uniq, let's use it to test whether + # coreutils is new enough (and GNU). + if [[ $(uniq --version 2>&1) == *"(GNU coreutils) "[6789]* ]]; then + CP="cp" + else + (bootstrap_coreutils) || return 1 + fi + fi + + # But for e.g. BSD, it isn't going to be, so if our test failed, + # use bootstrapped coreutils. + [[ -z ${CP} ]] && CP="${ROOT}/tmp/bin/cp" + [[ -x ${ROOT}/tmp/usr/bin/find ]] \ || [[ $(find --version 2>&1) == *GNU* ]] \ || (bootstrap_findutils) || return 1 @@ -1575,9 +1634,16 @@ bootstrap_stage1() { (bootstrap_tree) || return 1 # setup a profile - [[ -e ${ROOT}/etc/portage/make.profile && -e ${ROOT}/etc/portage/make.conf ]] || (bootstrap_setup) || return 1 + [[ -e ${ROOT}/etc/portage/make.profile && \ + -e ${MAKE_CONF_DIR}/0100_bootstrap_prefix_make.conf ]] \ + || (bootstrap_setup) || return 1 + + # setup a profile for stage2 mkdir -p "${ROOT}"/tmp/etc/. || return 1 - [[ -e ${ROOT}/tmp/etc/portage/make.profile ]] || cp -dpR "${ROOT}"/etc/portage "${ROOT}"/tmp/etc || return 1 + [[ -e ${ROOT}/tmp/etc/portage/make.profile ]] || \ + ( "${CP}" -dpR "${ROOT}"/etc/portage "${ROOT}"/tmp/etc && \ + rm -f "${ROOT}"/tmp/etc/portage/make.profile && \ + (ROOT="${ROOT}"/tmp PREFIX_DISABLE_RAP=yes bootstrap_profile) ) || return 1 # setup portage [[ -e ${ROOT}/tmp/usr/bin/emerge ]] || (bootstrap_portage) || return 1 @@ -1588,7 +1654,7 @@ bootstrap_stage1() { } bootstrap_stage1_log() { - bootstrap_stage1 ${@} 2>&1 | tee -a ${ROOT}/stage1.log + bootstrap_stage1 "${@}" 2>&1 | tee -a ${ROOT}/stage1.log local ret=${PIPESTATUS[0]} [[ ${ret} == 0 ]] && touch ${ROOT}/.stage1-finished return ${ret} @@ -1644,16 +1710,20 @@ do_emerge_pkgs() { clang internal-glib ) + local override_make_conf_dir="${PORTAGE_OVERRIDE_EPREFIX}${MAKE_CONF_DIR#${ROOT}}" + if [[ " ${USE} " == *" prefix-stack "* ]] && [[ ${PORTAGE_OVERRIDE_EPREFIX} == */tmp ]] && - ! grep -q '^USE=".*" # by bootstrap-prefix.sh$' "${PORTAGE_OVERRIDE_EPREFIX}/etc/portage/make.conf" + ! grep -Rq '^USE=".*" # by bootstrap-prefix.sh$' \ + "${override_make_conf_dir}" then # With prefix-stack, the USE env var does apply to the stacked # prefix only, not the base prefix (any more? since some portage # version?), so we have to persist the base USE flags into the # base prefix - without the additional incoming USE flags. + mkdir -p -- "${override_make_conf_dir}" echo "USE=\"\${USE} ${myuse[*]}\" # by bootstrap-prefix.sh" \ - >> "${PORTAGE_OVERRIDE_EPREFIX}/etc/portage/make.conf" + >> "${override_make_conf_dir}/0101_bootstrap_prefix_stack.conf" fi myuse=" ${myuse[*]} " local use @@ -1680,7 +1750,6 @@ do_emerge_pkgs() { && export CFLAGS=${OVERRIDE_CFLAGS} [[ -n ${OVERRIDE_CXXFLAGS} ]] \ && export CXXFLAGS=${OVERRIDE_CXXFLAGS} - PORTAGE_CONFIGROOT="${EPREFIX}" \ PORTAGE_SYNC_STALE=0 \ FEATURES="-news ${FEATURES}" \ USE="${myuse[*]}" \ @@ -1703,6 +1772,8 @@ do_emerge_pkgs() { } bootstrap_stage2() { + export PORTAGE_CONFIGROOT="${ROOT}"/tmp + if ! type -P emerge > /dev/null ; then eerror "emerge not found, did you bootstrap stage1?" return 1 @@ -1787,7 +1858,7 @@ bootstrap_stage2() { app-shells/bash app-arch/xz-utils sys-apps/sed - sys-apps/baselayout-prefix + sys-apps/baselayout dev-libs/libffi sys-devel/m4 sys-devel/flex @@ -1825,11 +1896,15 @@ bootstrap_stage2() { for pkg in ${compiler_stage1} ; do # > "${ROOT}"/etc/portage/make.conf + } >> "${MAKE_CONF_DIR}/0100_bootstrap_prefix_clang.conf" + # llvm won't setup symlinks to CHOST-clang here because # we're in a cross-ish situation (at least according to # multilib.eclass -- can't blame it at this point really) @@ -1872,13 +1949,15 @@ bootstrap_stage2() { } bootstrap_stage2_log() { - bootstrap_stage2 ${@} 2>&1 | tee -a ${ROOT}/stage2.log + bootstrap_stage2 "${@}" 2>&1 | tee -a ${ROOT}/stage2.log local ret=${PIPESTATUS[0]} [[ ${ret} == 0 ]] && touch "${ROOT}/.stage2-finished" return ${ret} } bootstrap_stage3() { + export PORTAGE_CONFIGROOT="${ROOT}" + if ! type -P emerge > /dev/null ; then eerror "emerge not found, did you bootstrap stage1?" return 1 @@ -1898,13 +1977,13 @@ bootstrap_stage3() { fi fi - # if we resume this stage and python-exec was installed already in + # If we resume this stage and python-exec was installed already in # tmp, we basically made the system unusable, so remove python-exec # here so we can use the python in tmp for pef in python{,3} python{,3}-config ; do rm -f "${ROOT}"/tmp/usr/bin/${pef} [[ ${pef} == *-config ]] && ppf=-config || ppf= - ( cd "${ROOT}"/tmp/usr/bin && ln -s python${PYTHONMAJMIN}${ppf} ${pef} ) + ( cd "${ROOT}"/tmp/usr/bin && ln -s python$(python_ver)${ppf} ${pef} ) done get_libdir() { @@ -1923,8 +2002,7 @@ bootstrap_stage3() { emerge_pkgs() { # stage3 tools should be used first. - # PORTAGE_TMPDIR, EMERGE_LOG_DIR, FEATURES=force-prefix are - # needed with host portage. + # PORTAGE_TMPDIR, EMERGE_LOG_DIR are needed with host portage. # # After the introduction of EAPI-7, eclasses now # strictly distinguish between build dependencies that @@ -1934,7 +2012,6 @@ bootstrap_stage3() { # PORTAGE_OVERRIDE_EPREFIX as BROOT is needed. PREROOTPATH="${ROOT}"$(echo /{,tmp/}{usr/,}{,lib/llvm/{12,11,10}/}{s,}bin | sed "s, ,:${ROOT},g") \ EPREFIX="${ROOT}" PORTAGE_TMPDIR="${PORTAGE_TMPDIR}" \ - FEATURES="${FEATURES} force-prefix" \ EMERGE_LOG_DIR="${ROOT}"/var/log \ STAGE=stage3 \ do_emerge_pkgs "$@" @@ -2031,8 +2108,12 @@ bootstrap_stage3() { ${linker} ) # use the new dynamic linker in place of rpath from now on. - RAP_DLINKER=$(echo "${ROOT}"/$(get_libdir)/ld*.so.[0-9]) - export LDFLAGS="-L${ROOT}/usr/$(get_libdir) -Wl,--dynamic-linker=${RAP_DLINKER}" + RAP_DLINKER=$(echo "${ROOT}"/$(get_libdir)/ld*.so.[0-9] | sed s"!${ROOT}/$(get_libdir)/ld-lsb.*!!") + export CPPFLAGS="--sysroot=${ROOT}" + export LDFLAGS="-Wl,--dynamic-linker=${RAP_DLINKER}" + # make sure these flags are used even in places that ignore/strip CPPFLAGS/LDFLAGS + export CC="gcc ${CPPFLAGS} ${LDFLAGS}" + export CXX="g++ ${CPPFLAGS} ${LDFLAGS}" BOOTSTRAP_RAP=yes \ pre_emerge_pkgs --nodeps "${pkgs[@]}" || return 1 @@ -2051,7 +2132,7 @@ bootstrap_stage3() { app-portage/elt-patches app-arch/xz-utils sys-apps/sed - sys-apps/baselayout-prefix + sys-apps/baselayout sys-devel/m4 sys-devel/flex sys-devel/binutils-config @@ -2076,36 +2157,37 @@ bootstrap_stage3() { # setup for a scenario where python doesn't live in the target # prefix and no helpers are available ( cd "${ROOT}"/usr/bin && test ! -e python && \ - ln -s "${ROOT}"/tmp/usr/bin/python${PYTHONMAJMIN} ) + ln -s "${ROOT}"/tmp/usr/bin/python$(python_ver) ) # in addition, avoid collisions - rm -Rf "${ROOT}"/tmp/usr/lib/python${PYTHONMAJMIN}/site-packages/clang + rm -Rf "${ROOT}"/tmp/usr/lib/python$(python_ver)/site-packages/clang - # try to get ourself out of the mud, bug #575324 + # Try to get ourself out of the mud, bug #575324 EXTRA_ECONF="--disable-compiler-version-checks $(rapx '--disable-lto --disable-bootstrap')" \ GCC_MAKE_TARGET=$(rapx all) \ MYCMAKEARGS="-DCMAKE_USE_SYSTEM_LIBRARY_LIBUV=OFF" \ - PYTHON_COMPAT_OVERRIDE=python${PYTHONMAJMIN} \ + PYTHON_COMPAT_OVERRIDE=python$(python_ver) \ pre_emerge_pkgs --nodeps ${compiler} || return 1 - # undo libgcc_s.so path of stage2 - # now we have the compiler right there - unset CXX CPPFLAGS LDFLAGS + # Undo libgcc_s.so path of stage2 + # Now we have the compiler right there + unset CC CXX CPPFLAGS LDFLAGS rm -f "${ROOT}"/etc/ld.so.conf.d/stage2.conf # need special care, it depends on texinfo, #717786 pre_emerge_pkgs --nodeps sys-apps/gawk || return 1 - ( cd "${ROOT}"/usr/bin && test ! -e python && rm -f python${PYTHONMAJMIN} ) + ( cd "${ROOT}"/usr/bin && test ! -e python && rm -f python$(python_ver) ) # Use $ROOT tools where possible from now on. if [[ $(readlink "${ROOT}"/bin/sh) == "${ROOT}/tmp/"* ]] ; then rm -f "${ROOT}"/bin/sh ln -s bash "${ROOT}"/bin/sh fi - # start using apps from new target + + # Start using apps from new target export PREROOTPATH="${ROOT}/usr/bin:${ROOT}/bin" - # get a sane bash, overwriting tmp symlinks + # Get a sane bash, overwriting tmp symlinks pre_emerge_pkgs "" "app-shells/bash" || return 1 # now we have a shell right there @@ -2123,7 +2205,8 @@ bootstrap_stage3() { app-admin/eselect $( [[ ${CHOST} == *-cygwin* ]] && echo sys-libs/cygwin-crypt ) ) - # for grep we need to do a little workaround as we might use llvm-3.4 + + # For grep we need to do a little workaround as we might use llvm-3.4 # here, which doesn't necessarily grok the system headers on newer # OSX, confusing the buildsystem ac_cv_c_decl_report=warning \ @@ -2161,19 +2244,16 @@ bootstrap_stage3() { emerge --color n --sync || emerge-webrsync || return 1 fi - # avoid installing git or encryption just for fun while completing @system + # Avoid installing git or encryption just for fun while completing @system export USE="-git -crypt" # Portage should figure out itself what it needs to do, if anything. - # Avoid glib compiling for Cocoa libs if it finds them, since we're - # still with an old llvm that may not understand the system headers - # very well on Darwin (-DGNUSTEP_BASE_VERSION hack) einfo "running emerge -uDNv system" estatus "stage3: emerge -uDNv system" - CPPFLAGS="-DGNUSTEP_BASE_VERSION" \ - CFLAGS= CXXFLAGS= emerge --color n -uDNv system || return 1 + unset CFLAGS CXXFLAGS CPPFLAGS + emerge --color n -uDNv system || return 1 - # remove anything that we don't need (compilers most likely) + # Remove anything that we don't need (compilers most likely) einfo "running emerge --depclean" estatus "stage3: emerge --depclean" emerge --color n --depclean @@ -2187,7 +2267,7 @@ bootstrap_stage3() { } bootstrap_stage3_log() { - bootstrap_stage3 ${@} 2>&1 | tee -a ${ROOT}/stage3.log + bootstrap_stage3 "${@}" 2>&1 | tee -a ${ROOT}/stage3.log local ret=${PIPESTATUS[0]} [[ ${ret} == 0 ]] && touch "${ROOT}/.stage3-finished" return ${ret} @@ -2198,10 +2278,11 @@ set_helper_vars() { export PORTDIR=${PORTDIR:-"${ROOT}/var/db/repos/gentoo"} export DISTDIR=${DISTDIR:-"${ROOT}/var/cache/distfiles"} PORTAGE_TMPDIR=${PORTAGE_TMPDIR:-${ROOT}/var/tmp} + MAKE_CONF_DIR="${ROOT}/etc/portage/make.conf/" DISTFILES_URL=${DISTFILES_URL:-"http://dev.gentoo.org/~grobian/distfiles"} GNU_URL=${GNU_URL:="http://ftp.gnu.org/gnu"} - DISTFILES_G_O="https://distfiles.prefix.bitzolder.nl" - DISTFILES_PFX="https://distfiles.prefix.bitzolder.nl/prefix" + DISTFILES_G_O="http://distfiles.prefix.bitzolder.nl" + DISTFILES_PFX="http://distfiles.prefix.bitzolder.nl/prefix" GENTOO_MIRRORS=${GENTOO_MIRRORS:="http://distfiles.gentoo.org"} SNAPSHOT_HOST=$(rapx ${DISTFILES_G_O} http://rsync.prefix.bitzolder.nl) SNAPSHOT_URL=${SNAPSHOT_URL:-"${SNAPSHOT_HOST}/snapshots"} @@ -2212,7 +2293,7 @@ set_helper_vars() { bootstrap_interactive() { # TODO should immediately die on platforms that we know are - # impossible due extremely hard dependency chains + # impossible due to extremely hard dependency chains # (NetBSD/OpenBSD) cat <<"EOF" @@ -2322,9 +2403,8 @@ EOF echo echo "It seems to me you are '${USER:-$(whoami 2> /dev/null)}' (${UID}), that looks cool to me." - # Expect noninteractive users to know what they do: - # Take EPREFIX from argv1 (=ROOT), not from env var. - [[ ${TODO} == 'noninteractive' ]] && EPREFIX=${ROOT} + # In case $ROOT were specified as $1, use it + [[ -z "${EPREFIX}" ]] && EPREFIX="${ROOT}" echo echo "I'm going to check for some variables in your environment now:" @@ -2555,6 +2635,7 @@ continue. Please execute: xcode-select -s /Library/Developer/CommandLineTools and try running me again. EOF + exit 1 fi else # let's see if we have an xcode install @@ -2566,18 +2647,25 @@ valid install. Try resetting it using: sudo xcode-select -r and try running me again. EOF + exit 1 fi fi fi echo local ncpu= case "${CHOST}" in - *-cygwin*) ncpu=$(cmd /D /Q /C 'echo %NUMBER_OF_PROCESSORS%' | tr -d "\\r") ;; - *-darwin*) ncpu=$(/usr/sbin/sysctl -n hw.ncpu) ;; - *-freebsd*) ncpu=$(/sbin/sysctl -n hw.ncpu) ;; - *-solaris*) ncpu=$(/usr/sbin/psrinfo | wc -l) ;; - *-linux-gnu*) ncpu=$(cat /proc/cpuinfo | grep processor | wc -l) ;; - *) ncpu=1 ;; + *-cygwin*) + ncpu=$(cmd /D /Q /C 'echo %NUMBER_OF_PROCESSORS%' | tr -d "\\r") ;; + *-darwin*) + ncpu=$(/usr/sbin/sysctl -n hw.ncpu) ;; + *-freebsd* | *-openbsd*) + ncpu=$(/sbin/sysctl -n hw.ncpu) ;; + *-solaris*) + ncpu=$(/usr/sbin/psrinfo | wc -l) ;; + *-linux-gnu*) + ncpu=$(cat /proc/cpuinfo | grep processor | wc -l) ;; + *) + ncpu=1 ;; esac # get rid of excess spaces (at least Solaris wc does) ncpu=$((ncpu + 0)) @@ -2835,8 +2923,8 @@ EOF # GNU and BSD variants of stat take different arguments (and # format specifiers are not equivalent) case "${CHOST}" in - *-darwin* | *-freebsd*) STAT='stat -f %u/%g' ;; - *) STAT='stat -c %U/%G' ;; + *-darwin* | *-freebsd* | *-openbsd*) STAT='stat -f %u/%g' ;; + *) STAT='stat -c %U/%G' ;; esac if [[ $(${STAT} "${EPREFIX}"/.canihaswrite) != \ @@ -3086,15 +3174,20 @@ if [[ -z ${CHOST} ]]; then if [[ x$(type -t uname) == "xfile" ]]; then case `uname -s` in Linux) + plt="gnu" + for f in /lib/ld-musl-*.so.1; do + [[ -e "$f" ]] && plt="musl" + done + sfx="unknown-linux-${plt}" case `uname -m` in ppc*) - CHOST="`uname -m | sed -e 's/^ppc/powerpc/'`-unknown-linux-gnu" + CHOST="`uname -m | sed -e 's/^ppc/powerpc/'`-${sfx}" ;; powerpc*|aarch64*) - CHOST="`uname -m`-unknown-linux-gnu" + CHOST="`uname -m`-${sfx}" ;; *) - CHOST="`uname -m`-pc-linux-gnu" + CHOST="`uname -m`-${sfx/unknown/pc}" ;; esac ;; @@ -3132,6 +3225,13 @@ if [[ -z ${CHOST} ]]; then ;; esac ;; + OpenBSD) + case `uname -m` in + amd64) + CHOST="x86_64-pc-openbsd`uname -r | sed 's|-.*$||'`" + ;; + esac + ;; *) eerror "Nothing known about platform `uname -s`." eerror "Please set CHOST appropriately for your system" @@ -3251,7 +3351,7 @@ then exit 1 fi -if [[ -n ${LD_LIBARY_PATH} || -n ${DYLD_LIBRARY_PATH} ]] ; then +if [[ -n ${LD_LIBRARY_PATH} || -n ${DYLD_LIBRARY_PATH} ]] ; then eerror "EEEEEK! You have LD_LIBRARY_PATH or DYLD_LIBRARY_PATH set" eerror "in your environment. This is a guarantee for TROUBLE." eerror "Cowardly refusing to operate any further this way!" diff --git a/bot/build.sh b/bot/build.sh new file mode 100755 index 00000000..507b79bf --- /dev/null +++ b/bot/build.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# +# script to build the EESSI compatibility layer. Intended use is that it is called +# by a (batch) job running on a compute node. +# +# This script is part of the EESSI compatibility layer, see +# https://github.com/EESSI/compatibility-layer.git +# +# author: Bob Droege (@bedroge) +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files referenced in job.cfg + +# stop as soon as something fails +set -e + +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} + +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} +echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'" + +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} +echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" + +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") +echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" + +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" + +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# obtain list of modules to be loaded +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") +echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" + +# load modules if LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/build.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/build.sh: no modules to be loaded" +fi + +##################################################### +# cpu_target_arch=$(echo ${CPU_TARGET} | cut -d/ -f1) +cpu_target_arch=$(cfg_get_value "architecture" "software_subdir" | cut -d/ -f1) +host_arch=$(uname -m) +eessi_arch=${cpu_target_arch:-${host_arch}} +eessi_os=linux +eessi_version=2023.03 +eessi_repo=pilot.eessi-hpc.org +tar_topdir=/cvmfs/${eessi_repo}/versions + +if [ "${eessi_arch}" != "${host_arch}" ]; then + echo "Requested architecture (${eessi_arch}) is different from this machine's architecture ($(uname -m))!" + exit 1 +fi + +#./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -c ~/compat.sif +./install_compatibility_layer.sh -a ${eessi_arch} -v ${eessi_version} -r ${eessi_repo} -g ${STORAGE} + +# create tarball -> should go into a separate script when this is supported by the bot +target_tgz=eessi-${eessi_version}-compat-linux-${eessi_arch}-$(date +%s).tar.gz +if [ -d ${eessi_tmp}/${tar_topdir}/${eessi_version} ]; then + echo ">> Creating tarball ${target_tgz} from ${eessi_tmp}/${tar_topdir}..." + tar cfvz ${target_tgz} -C ${eessi_tmp}/${tar_topdir} ${eessi_version}/compat/${eessi_os}/${eessi_arch} + echo ${target_tgz} created! +else + echo "Directory ${tar_topdir}/${eessi_version} was not created, not creating tarball." + exit 1 +fi diff --git a/install_compatibility_layer.sh b/install_compatibility_layer.sh new file mode 100755 index 00000000..7b28f1c2 --- /dev/null +++ b/install_compatibility_layer.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# +# Launch the compatibility layer installation using Ansible inside an Apptainer container. +# This can be run on any machine which has Apptainer installed, no special privileges are required. +# + +ARCH= +CONTAINER=docker://ghcr.io/eessi/bootstrap-prefix:debian11 +REPOSITORY="pilot.eessi-hpc.org" +RESUME= +RETAIN_TMP=0 +STORAGE= +VERSION= +VERBOSE= + +display_help() { + echo "usage: $0 [OPTIONS]" + echo "OPTIONS:" + echo " -a | --arch ARCHITECTURE" + echo " architecture to build a compatibility layer for" + echo " [default/required: current host's architecture]" + echo "" + echo " -c | --container IMAGE" + echo " image file or URL defining the container to use" + echo " [default: ${CONTAINER}]" + echo "" + echo " -g | --storage DIRECTORY" + echo " directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo "" + echo " -k | --retain-tmp" + echo " retain tmp storage (as tarball) for future" + echo " inspection [default: not set]" + echo "" + echo " -h | --help" + echo " display this usage information" + echo "" + echo " -r | --repository REPO" + echo " CVMFS repository name [default: ${REPOSITORY}]" + echo "" + echo " -t | --resume TMPDIR" + echo " tmp directory to resume from [default: None]" + echo "" + echo " -v | --version VERSION" + echo " override the EESSI stack version set in Ansible's" + echo " defaults/main.yml file [default: None]" + echo "" + echo " --verbose" + echo " increase verbosity of output [default: not set]" + echo +} + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -a|--arch) + ARCH="$2" + shift 2 + ;; + -c|--container) + CONTAINER="$2" + shift 2 + ;; + -g|--storage) + STORAGE="$2" + shift 2 + ;; + -k|--retain-tmp) + RETAIN_TMP=1 + shift 1 + ;; + -h|--help) + display_help + exit 0 + ;; + -r|--repository) + REPOSITORY="$2" + shift 2 + ;; + -t|--resume) + RESUME="$2" + shift 2 + ;; + -v|--version) + VERSION="$2" + shift 2 + ;; + --verbose) + VERBOSE="-vvv" + shift 1 + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +# We assume that this script is located in a directory containing a full checkout of the git repo, +# we verify this by checking for the existance oif the Ansible playbook. +SCRIPT_DIR=$(dirname $(realpath $0)) +if [ ! -f "${SCRIPT_DIR}/ansible/playbooks/install.yml" ]; then + echo "Ansible playbook install.yml cannot be found!" + echo "Make sure to run this script from a directory containing a the compatibility-layer git repository." + exit 1 +fi + +# source utils.sh (for get_container_runtime and check_exit_code) +source ${SCRIPT_DIR}/scripts/utils.sh + +# Check if the target architecture is set to the architecture of the current host, +# as that's the only thing that's currently supported by this script +HOST_ARCH=$(uname -m) +if [[ ! -z ${ARCH} ]] && [[ "${ARCH}" != "${HOST_ARCH}" ]]; then + echo "ERROR: this build host has architecture ${HOST_ARCH}, while a build for ${ARCH} was requested!" + exit 1 +fi +if [[ -z ${ARCH} ]]; then + ARCH=${HOST_ARCH} +fi +echo "A compatibility layer for architecture ${ARCH} will be built." + +# Make a temporary directory on the host for storing the installation and some temporary files +if [[ ! -z ${RESUME} ]] && [[ -d ${RESUME} ]]; then + EESSI_TMPDIR=${RESUME} + echo "using previous temporary storage at ${RESUME} to resume work" +else + TMPDIR=${STORAGE:-${TMPDIR:-/tmp}} + mkdir -p ${TMPDIR} + EESSI_TMPDIR=$(mktemp -d --tmpdir=${TMPDIR} eessi.XXXXXXXXXX) + echo "created new temporary storage at ${EESSI_TMPDIR}" +fi +echo "Using $EESSI_TMPDIR as temporary storage..." + +# Create temporary directories +mkdir -p ${EESSI_TMPDIR}/cvmfs +mkdir -p ${EESSI_TMPDIR}/home + +RUNTIME=$(get_container_runtime) +exit_code=$? +echo "RUNTIME='${RUNTIME}'" +check_exit_code ${exit_code} "using runtime ${RUNTIME}" "oh no, neither apptainer nor singularity available" + +# Set up paths and mount points for Apptainer +if [[ -z ${APPTAINER_CACHEDIR} ]]; then + export APPTAINER_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache +fi +export APPTAINER_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" +export APPTAINER_HOME="${EESSI_TMPDIR}/home:/home/${USER}" +# also define SINGULARITY_* env vars +if [[ -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/apptainer_cache +fi +export SINGULARITY_BIND="${EESSI_TMPDIR}/cvmfs:/cvmfs,${SCRIPT_DIR}:/compatibility-layer" +export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" + +# Construct the Ansible playbook command +ANSIBLE_OPTIONS="-e eessi_host_os=linux -e eessi_host_arch=$(uname -m)" +if [[ ! -z ${VERSION} ]]; then + ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} -e eessi_version=${VERSION}" +fi +if [[ ! -z ${REPOSITORY} ]]; then + ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} -e cvmfs_repository=${REPOSITORY}" +fi +if [[ ! -z ${VERBOSE} ]]; then + ANSIBLE_OPTIONS="${ANSIBLE_OPTIONS} ${VERBOSE}" +fi +ANSIBLE_COMMAND="ansible-playbook ${ANSIBLE_OPTIONS} /compatibility-layer/ansible/playbooks/install.yml" +# Finally, run Ansible inside the container to do the actual installation +echo "Executing ${ANSIBLE_COMMAND} in ${CONTAINER}, this will take a while..." +${RUNTIME} shell ${CONTAINER} < indexed array +# -A -> associative array +declare -A cfg_repos +declare -A cfg_file_map + + +# functions +function cfg_get_section { + if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then + echo ${BASH_REMATCH[2]} + else + echo "" + fi +} + +function cfg_get_key_value { + if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then + echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" + else + echo "" + fi +} + +function cfg_load { + local cur_section="" + local cur_key="" + local cur_val="" + IFS= + while read -r line; do + new_section=$(cfg_get_section $line) + # got a new section + if [[ -n "$new_section" ]]; then + cur_section=$new_section + # not a section, try a key value + else + val=$(cfg_get_key_value $line) + # trim leading and trailing spaces as well + cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) + cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) + if [[ -n "$cur_key" ]]; then + # section + key is the associative in bash array, the field separator is space + cfg_repos[${cur_section} ${cur_key}]=$cur_val + fi + fi + done <$1 +} + +function cfg_print { + for index in "${!cfg_repos[@]}" + do + # split the associative key in to section and key + echo -n "section : $(echo $index | cut -f1 -d ' ');" + echo -n "key : $(echo $index | cut -f2 -d ' ');" + echo "value: ${cfg_repos[$index]}" + done +} + +function cfg_sections { + declare -A sections + for key in "${!cfg_repos[@]}" + do + # extract section from the associative key + section=$(echo $key | cut -f1 -d ' ') + sections[${section}]=1 + done + for repo in "${!sections[@]}" + do + echo "${repo}" + done +} + +function cfg_get_value { + section=$1 + key=$2 + echo "${cfg_repos[$section $key]}" +} + +function cfg_trim_spaces { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< ${1} + else + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' < /dev/stdin + fi +} + +function cfg_trim_quotes { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^"*//' -e 's/"*$//' <<< ${1} + else + sed -e 's/^"*//' -e 's/"*$//' < /dev/stdin + fi +} + +function cfg_trim_curly_brackets { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^{*//' -e 's/}*$//' <<< ${1} + else + sed -e 's/^{*//' -e 's/}*$//' < /dev/stdin + fi +} + +function cfg_get_all_sections { + # first field in keys + # 1. get first field in all keys, 2. filter duplicates, 3. return them as string + declare -A all_sections + for key in "${!cfg_repos[@]}" + do + section=$(echo "$key" | cut -f1 -d' ') + all_sections[${section}]=1 + done + sections= + for sec_key in "${!all_sections[@]}" + do + sections="${sections} ${sec_key}" + done + echo "${sections}" | cfg_trim_spaces +} + +function cfg_init_file_map { + # strip '{' and '}' from config_map + # split config_map at ',' + # for each item: split at ':' use first as key, second as value + + # reset global variable + cfg_file_map=() + + # expects a string containing the config_map from the cfg file + # trim leading and trailing curly brackets + cm_trimmed=$(cfg_trim_curly_brackets "$1") + + # split into elements along ',' + declare -a cm_mappings + IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}" + + for index in "${!cm_mappings[@]}" + do + # split mapping into key and value + map_key=$(echo ${cm_mappings[index]} | cut -f1 -d':') + map_value=$(echo ${cm_mappings[index]} | cut -f2 -d':') + # trim spaces and double quotes at start and end + tr_key=$(cfg_trim_spaces "${map_key}" | cfg_trim_quotes) + tr_value=$(cfg_trim_spaces "${map_value}" | cfg_trim_quotes) + cfg_file_map[${tr_key}]=${tr_value} + done +} + +function cfg_print_map { + for index in "${!cfg_file_map[@]}" + do + echo "${index} --> ${cfg_file_map[${index}]}" + done +} diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100644 index 00000000..baec35f0 --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,117 @@ +function echo_green() { + echo -e "\e[32m$1\e[0m" +} + +function echo_red() { + echo -e "\e[31m$1\e[0m" +} + +function echo_yellow() { + echo -e "\e[33m$1\e[0m" +} + +ANY_ERROR_EXITCODE=1 +function fatal_error() { + echo_red "ERROR: $1" >&2 + if [[ $# -gt 1 ]]; then + exit $2 + else + exit "${ANY_ERROR_EXITCODE}" + fi +} + +function check_exit_code { + ec=$1 + ok_msg=$2 + fail_msg=$3 + + if [[ $ec -eq 0 ]]; then + echo_green "${ok_msg}" + else + fatal_error "${fail_msg}" + fi +} + +function get_path_for_tool { + tool_name=$1 + tool_envvar_name=$2 + + which_out=$(which ${tool_name} 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 + echo "${which_out}" + return 0 + fi + if [[ -z "${tool_envvar_name}" ]]; then + msg="no env var holding the full path to tool '${tool_name}' provided" + echo "${msg}" >&2 + return 1 + else + tool_envvar_value=${!tool_envvar_name} + if [[ -x "${tool_envvar_value}" ]]; then + msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" + echo "${msg}" >&2 + echo "${tool_envvar_value}" + return 0 + else + msg="ERROR: tool '${tool_name}' not in PATH\n" + msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" + echo "${msg}" >&2 + echo "" + return 2 + fi + fi +} + +function get_host_from_url { + url=$1 + re="(http|https)://([^/:]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_port_from_url { + url=$1 + re="(http|https)://[^:]+:([0-9]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_ipv4_address { + hname=$1 + hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + # TODO try other methods if the one above does not work --> tool that verifies + # what method can be used? + echo "${hipv4}" + return 0 +} + +# determine container runtime +function get_container_runtime { + apptainer_out=$(which apptainer 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "${apptainer_out}" + return 0 + fi + singularity_out=$(which singularity 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "${singularity_out}" + return 0 + else + echo "false" + return 1 + fi +}