From 2d8d500925fd90b7cc83b4fd8af3f53329a2696e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 11:24:55 +0100 Subject: [PATCH 1/8] ci-multinode: Use skc-ci-aio user for ci-multinode env Similar to c338dd9b7cad77c14eb15eb0193d02b0c9ff78b4, but applied to ci-multinode instead of ci-aio. This user only has read-only access to the package and container repositories, so is safer than using the release-train-ci user which has read/write permissions. --- .../environments/ci-multinode/stackhpc-ci.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index cdb6eb8109..ae5768bacc 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -11,6 +11,14 @@ kolla_docker_namespace: stackhpc-dev # Host and port of a package repository mirror. # Build and deploy the development Pulp service repositories. stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080" +stackhpc_repo_mirror_username: "skc-ci-aio" +stackhpc_repo_mirror_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 36373536303261313239613761653261663437356566343865383563346334396136653666383765 + 6634396534653865633936653038383132396532386665370a366562383166353966663838316266 + 65333133636330623936623438666632316238376264313234346333346461623765633163353635 + 6565326136313564320a303231383438333062643533333335663034613439393665656162626137 + 65356232656164663831316530333136336362393636656566353635306565626636 # Build and deploy released Pulp repository versions. stackhpc_repo_centos_stream_baseos_version: "{{ stackhpc_pulp_repo_centos_stream_8_baseos_version }}" @@ -66,12 +74,5 @@ stackhpc_include_os_minor_version_in_repo_url: true # Push built images to the development Pulp service registry. stackhpc_docker_registry: "{{ stackhpc_repo_mirror_url | regex_replace('^https?://', '') }}" -# Username and password of container registry. -stackhpc_docker_registry_username: "stackhpc-kayobe-ci" -stackhpc_docker_registry_password: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 33356166343730633865363431306535613736663764373034396132356131343066636530393534 - 3262646436663034633131316438633230383330633533350a386365313239303464383636376338 - 61656662333939333063343131633963636431663136643137636664633233633133396339613861 - 3038613063626138610a333566393937643630366564653163613364323965396130613433316537 - 39653335393831633362343934363866346262613166393561666336623062393935 +stackhpc_docker_registry_username: "{{ stackhpc_repo_mirror_username }}" +stackhpc_docker_registry_password: "{{ stackhpc_repo_mirror_password }}" From d77fcb1f64abbe50171d493df05e76e7f03e2a2e Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 15:29:00 +0100 Subject: [PATCH 2/8] ci-multinode: Use Ark package repositories to install packages Similar to e9130b9c51161fdadd676932eae5f2c13f5948a8 but applied to ci-multinode rather than ci-aio. Previously we were using Test Pulp on SMS lab, but this is out of action. Switching to Ark allows CI jobs to run on Leafcloud (or anywhere with Internet access). --- etc/kayobe/environments/ci-multinode/stackhpc-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml index ae5768bacc..32f8775e15 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc-ci.yml @@ -10,7 +10,8 @@ kolla_docker_namespace: stackhpc-dev # Host and port of a package repository mirror. # Build and deploy the development Pulp service repositories. -stackhpc_repo_mirror_url: "http://pulp-server.internal.sms-cloud:8080" +# Use Ark's package repositories to install packages. +stackhpc_repo_mirror_url: "{{ stackhpc_release_pulp_url }}" stackhpc_repo_mirror_username: "skc-ci-aio" stackhpc_repo_mirror_password: !vault | $ANSIBLE_VAULT;1.1;AES256 From c57f2c3a7b93d17ed1ccfd31c8c596dd6c2e3064 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 15:22:07 +0100 Subject: [PATCH 3/8] ci-multinode: Allow rebooting for SELinux state The Yoga overcloud host images currently have SELinux disabled, but the default config enables SELinux in permissive mode on Rocky Linux 9. This change allows the ci-multinode environment to run on these images. --- etc/kayobe/environments/ci-multinode/globals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/kayobe/environments/ci-multinode/globals.yml b/etc/kayobe/environments/ci-multinode/globals.yml index daecef4f23..fe7285f4ce 100644 --- a/etc/kayobe/environments/ci-multinode/globals.yml +++ b/etc/kayobe/environments/ci-multinode/globals.yml @@ -64,7 +64,7 @@ stackhpc_barbican_role_id_file_path: "/tmp/barbican-role-id" ############################################################################### # Avoid a reboot. -disable_selinux_do_reboot: false +disable_selinux_do_reboot: true ############################################################################### # Dummy variable to allow Ansible to accept this file. From e2b2f40cd138a212b2801ed410f791003a0a5fd5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 15:31:37 +0100 Subject: [PATCH 4/8] ci-multinode: Add API FQDNs to /etc/hosts in fix-networking.yml This avoids using the add-fqdn.yml playbook in terraform-kayobe-multinode, which requires the Terraform/Ansible client to have access to all hosts. --- etc/kayobe/ansible/fix-networking.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml index 0b14f9ddfa..8105db8f47 100644 --- a/etc/kayobe/ansible/fix-networking.yml +++ b/etc/kayobe/ansible/fix-networking.yml @@ -10,11 +10,13 @@ # Work around no known_hosts entry on first boot. ansible_ssh_common_args: "-o StrictHostKeyChecking=no" tasks: - - name: Ensure `hosts` file contains pulp entries + - name: Ensure `hosts` file contains pulp and API entries blockinfile: path: /etc/hosts - marker: "# {mark} Kayobe Pulp entries" + marker: "# {mark} Kayobe entries" block: | 10.0.0.34 pelican pelican.service.compute.sms-lab.cloud 10.205.3.187 pulp-server pulp-server.internal.sms-cloud + 192.168.37.2 internal.infra.mos.{{ root_domain }} + 192.168.39.2 public.infra.mos.{{ root_domain }} become: true From 2ca68f1713b5784fcfc577f37d3b6528f8017060 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Apr 2024 09:13:17 +0100 Subject: [PATCH 5/8] ci-multinode: Wait for connection in fix-networking.yml This allows us to drop the fix-homedir-ownership.yml playbook in terraform-kayobe-multinode, which also performed the function of waiting for hosts to become reachable. --- etc/kayobe/ansible/fix-networking.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etc/kayobe/ansible/fix-networking.yml b/etc/kayobe/ansible/fix-networking.yml index 8105db8f47..01a8332642 100644 --- a/etc/kayobe/ansible/fix-networking.yml +++ b/etc/kayobe/ansible/fix-networking.yml @@ -10,6 +10,9 @@ # Work around no known_hosts entry on first boot. ansible_ssh_common_args: "-o StrictHostKeyChecking=no" tasks: + - name: Ensure hosts are reachable + ansible.builtin.wait_for_connection: + - name: Ensure `hosts` file contains pulp and API entries blockinfile: path: /etc/hosts From 33c0d38622136cde0fd46458f837ecd35b18ac40 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Apr 2024 09:12:05 +0100 Subject: [PATCH 6/8] ci-multinode: Use qemu virtualisation Most multinode environments will use nested virtualisation, and we can't guarantee that nested KVM support is available. Use QEMU as a lowest common denominator. We might consider setting this dynamically based on the hypervisor in future. --- etc/kayobe/environments/ci-multinode/kolla/globals.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/kolla/globals.yml b/etc/kayobe/environments/ci-multinode/kolla/globals.yml index eab31a1d86..4f9506be09 100644 --- a/etc/kayobe/environments/ci-multinode/kolla/globals.yml +++ b/etc/kayobe/environments/ci-multinode/kolla/globals.yml @@ -1,4 +1,9 @@ --- +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu + # Reduce the control plane's memory footprint by limiting the number of worker # processes to two per-service when running in a VM. openstack_service_workers: "{% raw %}{{ [ansible_facts.processor_vcpus, 2 if ansible_facts.virtualization_role == 'guest' else 5] | min }}{% endraw %}" From eb1f88ec51ee65a504e064b99e3aadc4ab65cbe0 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 8 Apr 2024 16:03:51 +0100 Subject: [PATCH 7/8] ci-multinode: Set default Ceph release to Quincy on Rocky Linux 9 Pacific is not supported on Rocky Linux 9, so it does not make sense as a default. --- etc/kayobe/environments/ci-multinode/cephadm.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/etc/kayobe/environments/ci-multinode/cephadm.yml b/etc/kayobe/environments/ci-multinode/cephadm.yml index 7885a57359..4a9d3f4488 100644 --- a/etc/kayobe/environments/ci-multinode/cephadm.yml +++ b/etc/kayobe/environments/ci-multinode/cephadm.yml @@ -2,6 +2,12 @@ ############################################################################### # Cephadm deployment configuration. +# Ceph release name. +cephadm_ceph_release: "{{ 'quincy' if (ansible_facts['distribution_release'] == 'jammy' or ansible_facts.distribution_major_version == '9') else 'pacific' }}" + +# Ceph container image tag. +cephadm_image_tag: "{{ 'v17.2.7' if cephadm_ceph_release == 'quincy' else 'v16.2.14' }}" + # Ceph OSD specification. cephadm_osd_spec: service_type: osd From 50378160654a5ed5d74cb17180cb3999401d09b5 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Apr 2024 09:09:23 +0100 Subject: [PATCH 8/8] os_capacity: Add tags to playbook, update vault docs Previously the first deployment of a system with a Vault CA for internal TLS and os_capacity enabled would fail when deploying HAProxy. os_capacity deployment requires admin-openrc.sh to exist, but because of the use of -kt haproxy the post-deploy tasks that create it will be skipped. This change fixes the issue by adding an os_capacity tag to the relevant plays, and updating the Vault docs to skip the new tag when deploying HAProxy. --- doc/source/configuration/vault.rst | 2 +- etc/kayobe/ansible/deploy-os-capacity-exporter.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst index 4cb39b61b7..21268f1082 100644 --- a/doc/source/configuration/vault.rst +++ b/doc/source/configuration/vault.rst @@ -111,7 +111,7 @@ Setup HAProxy config for Vault .. code-block:: - kayobe overcloud service deploy -kt haproxy + kayobe overcloud service deploy --skip-tags os_capacity -kt haproxy Setup Vault HA on the overcloud hosts ------------------------------------- diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml index 978c13e62e..cc3afa7b0e 100644 --- a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml +++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml @@ -1,6 +1,7 @@ --- - name: Remove legacy os_exporter.cfg file hosts: network + tags: os_capacity gather_facts: false tasks: - name: Ensure legacy os_exporter.cfg config file is deleted @@ -11,6 +12,7 @@ - name: Deploy os-capacity exporter hosts: monitoring + tags: os_capacity gather_facts: false tasks: - name: Create os-capacity directory