From dc41cd937d5a3750bbfc40d69b5df36b4228d35d Mon Sep 17 00:00:00 2001 From: Ade Lee Date: Thu, 9 Apr 2026 17:01:58 +0000 Subject: [PATCH] [skmo] Add Skupper for cross-region RabbitMQ and Keystone internal routing Add hook playbooks and configuration to establish Skupper virtual services for RabbitMQ and Keystone internal endpoints, enabling cross-region connectivity in the multi-namespace SKMO scenario. skupper-connector.yaml: query the RabbitMQ TLS secret from the correct CRD - rabbitmq.openstack.org/v1beta1 (RabbitMq) as used by the OpenStack infra-operator, not the community rabbitmq.com/v1beta1 (RabbitmqCluster). Add retries to wait for spec.tls.secretName to be populated before creating the Skupper Connector. skupper-keystone-connector.yaml: add retries to the KeystoneAPI CR lookup to wait for spec.tls.api.internal.secretName to be available, since that field is not populated until Keystone completes TLS setup. configure-leaf-keystone-internal.yaml: after patching the leaf OSCP to use the Skupper Keystone virtual service, also create a MetalLB LoadBalancer Service (keystone-regionone-lb) and a DNSData CR (keystone-skupper) so that EDPM compute nodes outside the OCP cluster can resolve and connect to the Keystone auth_url. The Skupper Listener creates a ClusterIP-only Service that is unreachable from EDPM nodes; the LoadBalancer Service obtains a MetalLB IP on the leaf internalapi network and the DNSData entry registers both the short (.svc) and fully-qualified (.svc.cluster.local) names in the dnsmasq instance serving those nodes. Signed-off-by: Ade Lee Co-authored-by: Claude Made-with: Cursor --- .../configure-leaf-keystone-internal.yaml | 160 +++++++++++++++ .../skmo/configure-leaf-listener.yaml | 56 ++++-- hooks/playbooks/skmo/prepare-leaf.yaml | 25 +-- .../skmo/skupper-connector-tasks.yaml | 51 +++++ hooks/playbooks/skmo/skupper-connector.yaml | 58 ++++++ hooks/playbooks/skmo/skupper-install.yaml | 78 ++++++++ .../skmo/skupper-keystone-connector.yaml | 103 ++++++++++ hooks/playbooks/skmo/skupper-listener.yaml | 136 +++++++++++++ hooks/playbooks/skmo/skupper-sites.yaml | 189 ++++++++++++++++++ .../skmo/update-central-ca-bundle.yaml | 2 +- 10 files changed, 821 insertions(+), 37 deletions(-) create mode 100644 hooks/playbooks/skmo/configure-leaf-keystone-internal.yaml create mode 100644 hooks/playbooks/skmo/skupper-connector-tasks.yaml create mode 100644 hooks/playbooks/skmo/skupper-connector.yaml create mode 100644 hooks/playbooks/skmo/skupper-install.yaml create mode 100644 hooks/playbooks/skmo/skupper-keystone-connector.yaml create mode 100644 hooks/playbooks/skmo/skupper-listener.yaml create mode 100644 hooks/playbooks/skmo/skupper-sites.yaml diff --git a/hooks/playbooks/skmo/configure-leaf-keystone-internal.yaml b/hooks/playbooks/skmo/configure-leaf-keystone-internal.yaml new file mode 100644 index 000000000..87113b247 --- /dev/null +++ b/hooks/playbooks/skmo/configure-leaf-keystone-internal.yaml @@ -0,0 +1,160 @@ +--- +# Patch the leaf/workload region OpenStackControlPlane to use the Skupper +# Listener virtual endpoint for internal Keystone authentication traffic. +# +# The public endpoint override is left unchanged so that end-user traffic and +# the Keystone service catalog continue to reference the central region's +# external (public) URL. Only the *internal* override — used for all +# service-to-service communication inside the workload namespace — is updated +# to point at the Skupper Listener. +# +# After the OSCP is updated, this playbook also ensures that EDPM compute +# nodes can resolve the Skupper Keystone virtual service name. The Skupper +# Listener creates a ClusterIP-only Service (keystone-regionone) that is not +# reachable from outside the OCP cluster. EDPM nodes use the dnsmasq +# LoadBalancer Service (in the leaf namespace) as their DNS server and require +# a resolvable hostname for the Keystone auth_url configured in nova.conf. +# To bridge this gap, the playbook: +# 1. Creates a dedicated MetalLB LoadBalancer Service that selects the +# Skupper router pod and exposes port 5000 on the leaf internalapi network. +# 2. Creates a DNSData CR so dnsmasq resolves both the short (.svc) and +# fully-qualified (.svc.cluster.local) names to the LoadBalancer IP. +# +# This playbook is a no-op when cifmw_skupper_keystone_enabled is false. In +# that case, skupper-keystone-connector.yaml (pre_stage hook) has already +# written the public Keystone URL into skmo-values.yaml so the leaf OSCP is +# created with the public endpoint — no Skupper tunnel is used and there is no +# matching Connector for the Skupper Listener, so the OSCP must not be patched +# to point at the Skupper virtual Service. +# +# Run skupper-keystone-connector.yaml and skupper-keystone-listener.yaml +# before this playbook so that the Skupper virtual service is in place. +# +# Variables: +# cifmw_skupper_keystone_enabled (default: true) +# When false, all tasks in this playbook are skipped. +# cifmw_skupper_leaf_namespace (default: openstack2) +# cifmw_skupper_keystone_listener_host (default: keystone-regionone) +# cifmw_skupper_keystone_port (default: 5000) +# cifmw_skupper_keystone_metallb_pool (default: internalapi2) +# MetalLB address-pool name for the leaf internalapi network. An IP is +# auto-assigned from this pool; no static address is required. +- name: Configure leaf region to use Skupper Keystone internal endpoint + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + vars: + cifmw_skupper_keystone_enabled: true + cifmw_skupper_leaf_namespace: openstack2 + cifmw_skupper_keystone_listener_host: keystone-regionone + cifmw_skupper_keystone_port: 5000 + cifmw_skupper_keystone_metallb_pool: internalapi2 + tasks: + - name: Skip all tasks when Skupper Keystone routing is disabled + ansible.builtin.meta: end_play + when: not cifmw_skupper_keystone_enabled | bool + + - name: Build the Skupper Keystone internal URL + ansible.builtin.set_fact: + _skupper_keystone_internal_url: >- + https://{{ cifmw_skupper_keystone_listener_host }}.{{ cifmw_skupper_leaf_namespace }}.svc.cluster.local:{{ cifmw_skupper_keystone_port }} + + - name: Patch leaf OSCP internal Keystone override to use Skupper endpoint + # This switches the internal keystone endpoint URL from the central + # region's public URL to the Skupper Listener virtual service. The + # public endpoint override is not touched. + kubernetes.core.k8s: + state: patched + api_version: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + name: controlplane + namespace: "{{ cifmw_skupper_leaf_namespace }}" + definition: + spec: + keystone: + template: + override: + service: + internal: + endpointURL: "{{ _skupper_keystone_internal_url }}" + + - name: Wait for leaf OSCP to reconcile after Keystone endpoint change + kubernetes.core.k8s_info: + api_version: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + name: controlplane + namespace: "{{ cifmw_skupper_leaf_namespace }}" + register: _leaf_oscp + retries: 60 + delay: 30 + until: + - _leaf_oscp.resources | length > 0 + - _leaf_oscp.resources[0].status is defined + - _leaf_oscp.resources[0].status.conditions is defined + - _leaf_oscp.resources[0].status.conditions | + selectattr('type', 'equalto', 'Ready') | + selectattr('status', 'equalto', 'True') | list | length > 0 + + - name: Create LoadBalancer service to expose Skupper Keystone for EDPM nodes + # The Skupper Listener creates a ClusterIP-only Service that EDPM nodes + # outside the OCP cluster cannot reach. This LoadBalancer Service selects + # the same Skupper router pod and obtains a MetalLB IP on the leaf + # internalapi network, making port 5000 reachable from EDPM compute nodes. + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Service + metadata: + name: "{{ cifmw_skupper_keystone_listener_host }}-lb" + namespace: "{{ cifmw_skupper_leaf_namespace }}" + annotations: + metallb.universe.tf/address-pool: "{{ cifmw_skupper_keystone_metallb_pool }}" + spec: + type: LoadBalancer + selector: + application: skupper-router + skupper.io/component: router + ports: + - name: keystone-internal + port: "{{ cifmw_skupper_keystone_port | int }}" + protocol: TCP + targetPort: 1024 + + - name: Wait for MetalLB to assign an external IP to the keystone LoadBalancer + kubernetes.core.k8s_info: + api_version: v1 + kind: Service + name: "{{ cifmw_skupper_keystone_listener_host }}-lb" + namespace: "{{ cifmw_skupper_leaf_namespace }}" + register: _keystone_lb_svc + retries: 12 + delay: 10 + until: + - _keystone_lb_svc.resources | length > 0 + - _keystone_lb_svc.resources[0].status.loadBalancer.ingress is defined + - _keystone_lb_svc.resources[0].status.loadBalancer.ingress | length > 0 + + - name: Set keystone LoadBalancer IP fact + ansible.builtin.set_fact: + _keystone_lb_ip: >- + {{ _keystone_lb_svc.resources[0].status.loadBalancer.ingress[0].ip }} + + - name: Create DNSData entry for Skupper Keystone endpoint + # Adds both the short (.svc) and fully-qualified (.svc.cluster.local) + # names to the dnsmasq instance serving EDPM nodes, so that nova-compute + # auth_url lookups resolve to the LoadBalancer IP above. + kubernetes.core.k8s: + state: present + definition: + apiVersion: network.openstack.org/v1beta1 + kind: DNSData + metadata: + name: keystone-skupper + namespace: "{{ cifmw_skupper_leaf_namespace }}" + spec: + dnsDataLabelSelectorValue: dnsdata + hosts: + - hostnames: + - "{{ cifmw_skupper_keystone_listener_host }}.{{ cifmw_skupper_leaf_namespace }}.svc" + - "{{ cifmw_skupper_keystone_listener_host }}.{{ cifmw_skupper_leaf_namespace }}.svc.cluster.local" + ip: "{{ _keystone_lb_ip }}" diff --git a/hooks/playbooks/skmo/configure-leaf-listener.yaml b/hooks/playbooks/skmo/configure-leaf-listener.yaml index 2bfa8aee0..78f8cc26b 100644 --- a/hooks/playbooks/skmo/configure-leaf-listener.yaml +++ b/hooks/playbooks/skmo/configure-leaf-listener.yaml @@ -1,29 +1,57 @@ --- -- name: Patch leaf control plane with barbican-keystone-listener transport URL - hosts: localhost +# Configure the leaf barbican-keystone-listener to use the Skupper +# application network for cross-region RabbitMQ access. +# +# In the leaf region: +# - Read the RabbitMQ credentials from the dedicated user credentials secret +# created by the RabbitMQ operator when the TransportURL CR is reconciled. +# - Patch barbicanKeystoneListener to connect to the central RabbitMQ via the +# Skupper Listener endpoint using those credentials and its own pool_name. +# +# Variables: +# cifmw_skupper_central_namespace (default: openstack) +# cifmw_skupper_leaf_namespace (default: openstack2) +# cifmw_skupper_listener_host (default: rabbitmq-regionone) +# Must match the host set in skupper-listener.yaml. +# cifmw_skupper_rabbitmq_port (default: 5671) +# cifmw_skupper_transport_url_name (default: barbican-keystone-listener-regiontwo) +# Name of the TransportURL CR created in prepare-leaf.yaml. The operator +# creates a user credentials secret named: +# rabbitmq-user---user +# cifmw_skupper_transport_url_username (default: barbican-keystone-listener-regiontwo) +# Must match the username field set on the TransportURL CR in prepare-leaf.yaml. +- name: Configure barbican-keystone-listener to use Skupper for cross-region RabbitMQ + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" gather_facts: false vars: - central_namespace: openstack - leaf_namespace: openstack2 - leaf_transport_url_name: rabbitmq-transport-url-barbican-keystone-listener-regiontwo + cifmw_skupper_central_namespace: openstack + cifmw_skupper_leaf_namespace: openstack2 + cifmw_skupper_listener_host: rabbitmq-regionone + cifmw_skupper_rabbitmq_port: 5671 + cifmw_skupper_transport_url_name: barbican-keystone-listener-regiontwo + cifmw_skupper_transport_url_username: barbican-keystone-listener-regiontwo tasks: - - name: Get transport URL secret from central namespace + - name: Get RabbitMQ user credentials secret for leaf listener + # The RabbitMQ operator creates a secret named + # rabbitmq-user---user that contains + # the username and password fields for the dedicated RabbitMQ user. kubernetes.core.k8s_info: api_version: v1 kind: Secret - namespace: "{{ central_namespace }}" - name: "{{ leaf_transport_url_name }}" - register: _transport_secret + namespace: "{{ cifmw_skupper_central_namespace }}" + name: "rabbitmq-user-{{ cifmw_skupper_transport_url_name }}-{{ cifmw_skupper_transport_url_username }}-user" + register: _rabbitmq_user_secret - - name: Patch OpenStackControlPlane in leaf region with notifications transport_url + - name: Patch leaf barbicanKeystoneListener to use Skupper RabbitMQ endpoint vars: - _transport_url: "{{ _transport_secret.resources[0].data['transport_url'] | b64decode }}" + _username: "{{ _rabbitmq_user_secret.resources[0].data['username'] | b64decode }}" + _password: "{{ _rabbitmq_user_secret.resources[0].data['password'] | b64decode }}" kubernetes.core.k8s: state: patched api_version: core.openstack.org/v1beta1 kind: OpenStackControlPlane name: controlplane - namespace: "{{ leaf_namespace }}" + namespace: "{{ cifmw_skupper_leaf_namespace }}" definition: spec: barbican: @@ -31,6 +59,6 @@ barbicanKeystoneListener: customServiceConfig: | [DEFAULT] - transport_url = {{ _transport_url }} + transport_url = rabbit://{{ _username }}:{{ _password }}@{{ cifmw_skupper_listener_host }}:{{ cifmw_skupper_rabbitmq_port }}/?ssl=1 [keystone_notifications] - pool_name = barbican-listener-regionTwo + pool_name = barbican-listener-regiontwo diff --git a/hooks/playbooks/skmo/prepare-leaf.yaml b/hooks/playbooks/skmo/prepare-leaf.yaml index 8e0f3a706..43c24a92d 100644 --- a/hooks/playbooks/skmo/prepare-leaf.yaml +++ b/hooks/playbooks/skmo/prepare-leaf.yaml @@ -1,6 +1,6 @@ --- - name: Prepare SKMO leaf prerequisites in regionZero - hosts: localhost + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" gather_facts: false vars: skmo_values_file: "{{ cifmw_architecture_repo }}/examples/va/multi-namespace-skmo/control-plane2/skmo-values.yaml" @@ -10,8 +10,8 @@ central_rootca_secret: rootca-public central_rootca_internal_secret: rootca-internal leaf_transport_url_name: barbican-keystone-listener-regiontwo + leaf_transport_url_username: barbican-keystone-listener-regiontwo leaf_transport_url_name_secret: rabbitmq-transport-url-barbican-keystone-listener-regiontwo - leaf_transport_url_secret_copy: barbican-keystone-listener-regiontwo-transport tasks: - name: Wait for central Keystone API to be ready kubernetes.core.k8s_info: @@ -159,6 +159,7 @@ namespace: "{{ central_namespace }}" spec: rabbitmqClusterName: rabbitmq + username: "{{ leaf_transport_url_username }}" - name: Wait for TransportURL to be ready kubernetes.core.k8s_info: @@ -176,23 +177,3 @@ - _transport_url_info.resources[0].status.conditions | selectattr('type', 'equalto', 'Ready') | selectattr('status', 'equalto', 'True') | list | length > 0 - - - name: Get transport URL secret from central namespace - kubernetes.core.k8s_info: - api_version: v1 - kind: Secret - namespace: "{{ central_namespace }}" - name: "{{ leaf_transport_url_name_secret }}" - register: _transport_secret - - - name: Copy transport URL secret to leaf namespace - kubernetes.core.k8s: - state: present - definition: - apiVersion: v1 - kind: Secret - metadata: - name: "{{ leaf_transport_url_secret_copy }}" - namespace: "{{ leaf_namespace }}" - type: "{{ _transport_secret.resources[0].type }}" - data: "{{ _transport_secret.resources[0].data }}" diff --git a/hooks/playbooks/skmo/skupper-connector-tasks.yaml b/hooks/playbooks/skmo/skupper-connector-tasks.yaml new file mode 100644 index 000000000..b09e3fe98 --- /dev/null +++ b/hooks/playbooks/skmo/skupper-connector-tasks.yaml @@ -0,0 +1,51 @@ +--- +# Shared task file: Create a Skupper Connector CR and wait for it to be +# Configured. Include this from service-specific connector playbooks after +# the TLS credentials have been discovered and stored in the variables below. +# +# Expected variables (set via include_tasks vars: block): +# _cifmw_connector_name Skupper Connector CR name +# _cifmw_connector_namespace Namespace for the Connector +# _cifmw_connector_routing_key Skupper routing key (must match Listener) +# _cifmw_connector_host Backend service hostname +# _cifmw_connector_port Backend service port +# _cifmw_connector_tls_credentials Name of the TLS Secret for the backend +# _cifmw_connector_verify_hostname Whether Skupper verifies the backend cert hostname +# _cifmw_connector_ignore_wait_errors Whether to ignore wait failures + +- name: Create Skupper Connector + kubernetes.core.k8s: + state: present + definition: + apiVersion: skupper.io/v2alpha1 + kind: Connector + metadata: + name: "{{ _cifmw_connector_name }}" + namespace: "{{ _cifmw_connector_namespace }}" + spec: + routingKey: "{{ _cifmw_connector_routing_key }}" + host: "{{ _cifmw_connector_host }}" + port: "{{ _cifmw_connector_port }}" + type: tcp + tlsCredentials: "{{ _cifmw_connector_tls_credentials }}" + verifyHostname: "{{ _cifmw_connector_verify_hostname }}" + +- name: Wait for Skupper Connector to be configured + # A Connector shows "Ready" only after a matching Listener is deployed in + # the remote namespace. Waiting for "Configured" is sufficient here. + ignore_errors: "{{ _cifmw_connector_ignore_wait_errors | bool }}" + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Connector + name: "{{ _cifmw_connector_name }}" + namespace: "{{ _cifmw_connector_namespace }}" + register: _connector + retries: 30 + delay: 10 + until: + - _connector.resources | length > 0 + - _connector.resources[0].status is defined + - _connector.resources[0].status.conditions is defined + - _connector.resources[0].status.conditions | + selectattr('type', 'equalto', 'Configured') | + selectattr('status', 'equalto', 'True') | list | length > 0 diff --git a/hooks/playbooks/skmo/skupper-connector.yaml b/hooks/playbooks/skmo/skupper-connector.yaml new file mode 100644 index 000000000..f372ac338 --- /dev/null +++ b/hooks/playbooks/skmo/skupper-connector.yaml @@ -0,0 +1,58 @@ +--- +# Create a Skupper Connector in the central namespace that exposes the central +# RabbitMQ service to workload regions over the Skupper application network. +# +# The TLS Secret name is auto-discovered from the RabbitmqCluster CR. +# Connector creation and the wait for Configured status are handled by the +# shared skupper-connector-tasks.yaml task file. +# +# Variables: +# cifmw_skupper_central_namespace (default: openstack) +# cifmw_skupper_routing_key (default: rabbit-keystone) +# cifmw_skupper_rabbitmq_port (default: 5671) +- name: Create Skupper Connector for central RabbitMQ + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + vars: + cifmw_skupper_central_namespace: openstack + cifmw_skupper_routing_key: rabbit-keystone + cifmw_skupper_rabbitmq_port: 5671 + tasks: + - name: Check if Skupper Connector already exists + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Connector + name: rabbitmq-keystone + namespace: "{{ cifmw_skupper_central_namespace }}" + register: _existing_connector + + - name: Get RabbitMQ TLS certificate secret name from RabbitMq CR + # The OpenStack infra-operator uses rabbitmq.openstack.org/v1beta1 (RabbitMq), + # not the community rabbitmq.com/v1beta1 (RabbitmqCluster). + when: _existing_connector.resources | length == 0 + kubernetes.core.k8s_info: + api_version: rabbitmq.openstack.org/v1beta1 + kind: RabbitMq + name: rabbitmq + namespace: "{{ cifmw_skupper_central_namespace }}" + register: _rabbitmq_cluster + retries: 30 + delay: 10 + until: + - _rabbitmq_cluster.resources | length > 0 + - _rabbitmq_cluster.resources[0].spec.tls is defined + - _rabbitmq_cluster.resources[0].spec.tls.secretName is defined + - _rabbitmq_cluster.resources[0].spec.tls.secretName | length > 0 + + - name: Create Skupper Connector and wait for Configured + when: _existing_connector.resources | length == 0 + ansible.builtin.include_tasks: skupper-connector-tasks.yaml + vars: + _cifmw_connector_name: rabbitmq-keystone + _cifmw_connector_namespace: "{{ cifmw_skupper_central_namespace }}" + _cifmw_connector_routing_key: "{{ cifmw_skupper_routing_key }}" + _cifmw_connector_host: "rabbitmq.{{ cifmw_skupper_central_namespace }}.svc.cluster.local" + _cifmw_connector_port: "{{ cifmw_skupper_rabbitmq_port }}" + _cifmw_connector_tls_credentials: "{{ _rabbitmq_cluster.resources[0].spec.tls.secretName }}" + _cifmw_connector_verify_hostname: true + _cifmw_connector_ignore_wait_errors: false diff --git a/hooks/playbooks/skmo/skupper-install.yaml b/hooks/playbooks/skmo/skupper-install.yaml new file mode 100644 index 000000000..c03a34254 --- /dev/null +++ b/hooks/playbooks/skmo/skupper-install.yaml @@ -0,0 +1,78 @@ +--- +# Install the Skupper operator (cluster-scope) on the current cluster. +# +# Variables: +# cifmw_skupper_install_source (default: upstream) +# upstream - apply the upstream install YAML from skupper.io +# downstream - apply a locally-downloaded Red Hat Service Interconnect YAML +# cifmw_skupper_upstream_install_url (default: https://skupper.io/v2/install.yaml) +# cifmw_skupper_downstream_install_file (no default; required when source=downstream) +- name: Install Skupper operator + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + vars: + cifmw_skupper_install_source: upstream + cifmw_skupper_upstream_install_url: "https://skupper.io/v2/install.yaml" + cifmw_skupper_downstream_install_file: "" + tasks: + - name: Check if Skupper CRD is already present + kubernetes.core.k8s_info: + api_version: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + name: sites.skupper.io + register: _skupper_crd + + - name: Fetch upstream Skupper install YAML + when: + - _skupper_crd.resources | length == 0 + - cifmw_skupper_install_source == 'upstream' + ansible.builtin.uri: + url: "{{ cifmw_skupper_upstream_install_url }}" + return_content: true + register: _skupper_upstream_content + + - name: Apply upstream Skupper install YAML + when: + - _skupper_crd.resources | length == 0 + - cifmw_skupper_install_source == 'upstream' + kubernetes.core.k8s: + state: present + definition: "{{ item }}" + loop: "{{ _skupper_upstream_content.content | from_yaml_all | select('ne', None) | list }}" + + - name: Apply downstream Skupper install YAML (Red Hat Service Interconnect) + when: + - _skupper_crd.resources | length == 0 + - cifmw_skupper_install_source == 'downstream' + ansible.builtin.assert: + that: cifmw_skupper_downstream_install_file | length > 0 + fail_msg: >- + cifmw_skupper_downstream_install_file must be set when + cifmw_skupper_install_source is 'downstream'. + + - name: Apply downstream install file + when: + - _skupper_crd.resources | length == 0 + - cifmw_skupper_install_source == 'downstream' + - cifmw_skupper_downstream_install_file | length > 0 + kubernetes.core.k8s: + state: present + src: "{{ cifmw_skupper_downstream_install_file }}" + + - name: Wait for Skupper controller to be ready + # The upstream install names the deployment "skupper-controller" while the + # downstream Red Hat Service Interconnect install uses + # "skupper-controller-manager". List all Deployments in the namespace and + # wait until at least one is fully ready, regardless of name. + kubernetes.core.k8s_info: + api_version: apps/v1 + kind: Deployment + namespace: skupper + register: _skupper_deploy + retries: 30 + delay: 10 + until: + - _skupper_deploy.resources | length > 0 + - _skupper_deploy.resources | + selectattr('status.readyReplicas', 'defined') | + selectattr('status.readyReplicas', 'ge', 1) | list | length > 0 diff --git a/hooks/playbooks/skmo/skupper-keystone-connector.yaml b/hooks/playbooks/skmo/skupper-keystone-connector.yaml new file mode 100644 index 000000000..6a6b35baa --- /dev/null +++ b/hooks/playbooks/skmo/skupper-keystone-connector.yaml @@ -0,0 +1,103 @@ +--- +# Create a Skupper Connector in the central namespace that exposes the internal +# Keystone service endpoint to workload regions over the Skupper application +# network. +# +# In a standard SKMO deployment the workload regions reach the central +# Keystone service through its *public* (external) endpoint. By creating a +# Skupper Connector here and a matching Listener in each workload namespace +# (via skupper-listener.yaml), all service-to-service authentication traffic +# from a workload region travels over the mTLS Skupper tunnel instead of the +# public internet. +# +# This playbook also rewrites keystoneInternalURL in skmo-values.yaml to the +# correct URL (Skupper virtual Service or public endpoint) before the kustomize +# build runs, so the leaf OSCP is created with the right endpoint from the +# first apply — no rolling restart required (Option A). +# +# Connector creation and the wait for Configured status are handled by the +# shared skupper-connector-tasks.yaml task file. +# +# Variables: +# cifmw_skupper_central_namespace (default: openstack) +# cifmw_skupper_keystone_routing_key (default: keystone-internal) +# cifmw_skupper_keystone_port (default: 5000) +# cifmw_skupper_keystone_cert_secret (default: "") +# TLS Secret name for the Keystone backend. Auto-discovered from the +# KeystoneAPI CR (spec.tls.api.internal.secretName) when empty. +# cifmw_skupper_keystone_enabled (default: true) +# When true, keystoneInternalURL is set to the Skupper virtual Service URL +# and the Connector is created. When false, keystoneInternalURL is left at +# the public URL and all Skupper tasks are skipped. +# cifmw_skupper_keystone_internal_url (default: https://keystone-regionone.openstack2.svc.cluster.local:5000) +# cifmw_skupper_keystone_public_url (default: https://keystone-public-openstack.apps.ocp.openstack.lab) +- name: Create Skupper Connector for central Keystone internal endpoint + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + vars: + cifmw_skupper_central_namespace: openstack + cifmw_skupper_keystone_routing_key: keystone-internal + cifmw_skupper_keystone_port: 5000 + cifmw_skupper_keystone_cert_secret: "" + cifmw_skupper_keystone_enabled: true + cifmw_skupper_keystone_internal_url: "https://keystone-regionone.openstack2.svc.cluster.local:5000" + cifmw_skupper_keystone_public_url: "https://keystone-public-openstack.apps.ocp.openstack.lab" + _skmo_values_file: "{{ cifmw_architecture_repo }}/examples/va/multi-namespace-skmo/control-plane2/skmo-values.yaml" + tasks: + - name: Set keystoneInternalURL in skmo-values.yaml based on Skupper flag + # Both values are constants — no discovery needed. This task runs before + # the kustomize build so the correct URL is baked into the leaf OSCP from + # its first apply, avoiding a rolling restart (Option A). + ansible.builtin.lineinfile: + path: "{{ _skmo_values_file }}" + regexp: '^\s+keystoneInternalURL:' + line: " keystoneInternalURL: {{ cifmw_skupper_keystone_internal_url if cifmw_skupper_keystone_enabled | bool else cifmw_skupper_keystone_public_url }}" + + - name: Get KeystoneAPI CR to discover internal TLS cert secret name + when: + - cifmw_skupper_keystone_enabled | bool + - cifmw_skupper_keystone_cert_secret | length == 0 + kubernetes.core.k8s_info: + api_version: keystone.openstack.org/v1beta1 + kind: KeystoneAPI + name: keystone + namespace: "{{ cifmw_skupper_central_namespace }}" + register: _keystone_api + retries: 60 + delay: 10 + until: + - _keystone_api.resources | length > 0 + - _keystone_api.resources[0].spec.tls is defined + - _keystone_api.resources[0].spec.tls.api is defined + - _keystone_api.resources[0].spec.tls.api.internal is defined + - _keystone_api.resources[0].spec.tls.api.internal.secretName is defined + + - name: Set Keystone TLS secret fact from KeystoneAPI CR + when: + - cifmw_skupper_keystone_enabled | bool + - cifmw_skupper_keystone_cert_secret | length == 0 + ansible.builtin.set_fact: + _keystone_tls_secret: >- + {{ _keystone_api.resources[0].spec.tls.api.internal.secretName }} + + - name: Set Keystone TLS secret fact from variable override + when: + - cifmw_skupper_keystone_enabled | bool + - cifmw_skupper_keystone_cert_secret | length > 0 + ansible.builtin.set_fact: + _keystone_tls_secret: "{{ cifmw_skupper_keystone_cert_secret }}" + + - name: Create Skupper Connector and wait for Configured + when: cifmw_skupper_keystone_enabled | bool + ansible.builtin.include_tasks: skupper-connector-tasks.yaml + vars: + _cifmw_connector_name: keystone-internal + _cifmw_connector_namespace: "{{ cifmw_skupper_central_namespace }}" + _cifmw_connector_routing_key: "{{ cifmw_skupper_keystone_routing_key }}" + _cifmw_connector_host: "keystone-internal.{{ cifmw_skupper_central_namespace }}.svc.cluster.local" + _cifmw_connector_port: "{{ cifmw_skupper_keystone_port }}" + _cifmw_connector_tls_credentials: "{{ _keystone_tls_secret }}" + # verifyHostname is false because the cluster-internal service name may + # differ from the SANs in the Keystone TLS certificate. + _cifmw_connector_verify_hostname: false + _cifmw_connector_ignore_wait_errors: false diff --git a/hooks/playbooks/skmo/skupper-listener.yaml b/hooks/playbooks/skmo/skupper-listener.yaml new file mode 100644 index 000000000..4dabf3c52 --- /dev/null +++ b/hooks/playbooks/skmo/skupper-listener.yaml @@ -0,0 +1,136 @@ +--- +# Generic Skupper Listener playbook. +# +# Creates a cert-manager Certificate and a Skupper Listener CR in a workload +# namespace, exposing a remote service (from the central namespace) as a +# cluster-local virtual Service over the Skupper mTLS application network. +# +# Invoke this playbook from the automation vars with extra_vars that supply +# the service-specific values. All variables have empty or conservative +# defaults; required variables are noted below. +# +# Variables: +# cifmw_skupper_listener_namespace (default: openstack2) +# cifmw_skupper_listener_name (required) Skupper Listener CR name +# cifmw_skupper_listener_cert_name (required) cert-manager Certificate CR name +# cifmw_skupper_listener_cert_secret (required) Secret name for the TLS cert +# cifmw_skupper_listener_routing_key (required) Must match the Connector routing key +# cifmw_skupper_listener_host (required) Hostname for the virtual Service +# cifmw_skupper_listener_port (required) Port for the virtual Service +# cifmw_skupper_listener_ignore_wait_errors (default: false) +# Set to true when running in pre_stage_run before the leaf OSCP is +# deployed. The cert-manager Certificate will be pending until the OSCP +# operator creates the rootca-internal Issuer; the Skupper Listener virtual +# Service is still created immediately, and TLS credentials are picked up +# automatically once cert-manager issues the certificate. The leaf OSCP's +# wait condition provides the convergence guarantee. +# cifmw_skupper_listener_enabled (default: true) +# Set to false to skip all tasks. Pass this alongside any feature flag +# (e.g. cifmw_skupper_keystone_enabled) that controls whether this Listener +# should be created at all. +- name: Create Skupper Listener + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + vars: + cifmw_skupper_listener_namespace: openstack2 + cifmw_skupper_listener_name: "" + cifmw_skupper_listener_cert_name: "" + cifmw_skupper_listener_cert_secret: "" + cifmw_skupper_listener_routing_key: "" + cifmw_skupper_listener_host: "" + cifmw_skupper_listener_port: "" + cifmw_skupper_listener_ignore_wait_errors: false + cifmw_skupper_listener_enabled: true + tasks: + - name: Check if Skupper Listener already exists + when: cifmw_skupper_listener_enabled | bool + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Listener + name: "{{ cifmw_skupper_listener_name }}" + namespace: "{{ cifmw_skupper_listener_namespace }}" + register: _existing_listener + + - name: Create cert-manager Certificate for Skupper Listener + when: + - cifmw_skupper_listener_enabled | bool + - _existing_listener.resources | length == 0 + kubernetes.core.k8s: + state: present + definition: + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: "{{ cifmw_skupper_listener_cert_name }}" + namespace: "{{ cifmw_skupper_listener_namespace }}" + spec: + secretName: "{{ cifmw_skupper_listener_cert_secret }}" + issuerRef: + name: rootca-internal + kind: Issuer + dnsNames: + - "{{ cifmw_skupper_listener_host }}" + - "{{ cifmw_skupper_listener_host }}.{{ cifmw_skupper_listener_namespace }}.svc" + - "{{ cifmw_skupper_listener_host }}.{{ cifmw_skupper_listener_namespace }}.svc.cluster.local" + usages: + - digital signature + - key encipherment + - server auth + + - name: Wait for Listener certificate Secret to be created + when: + - cifmw_skupper_listener_enabled | bool + - _existing_listener.resources | length == 0 + ignore_errors: "{{ cifmw_skupper_listener_ignore_wait_errors | bool }}" + kubernetes.core.k8s_info: + api_version: v1 + kind: Secret + name: "{{ cifmw_skupper_listener_cert_secret }}" + namespace: "{{ cifmw_skupper_listener_namespace }}" + register: _listener_cert_secret + retries: 12 + delay: 10 + until: _listener_cert_secret.resources | length > 0 + + - name: Create Skupper Listener + when: + - cifmw_skupper_listener_enabled | bool + - _existing_listener.resources | length == 0 + kubernetes.core.k8s: + state: present + definition: + apiVersion: skupper.io/v2alpha1 + kind: Listener + metadata: + name: "{{ cifmw_skupper_listener_name }}" + namespace: "{{ cifmw_skupper_listener_namespace }}" + spec: + routingKey: "{{ cifmw_skupper_listener_routing_key }}" + host: "{{ cifmw_skupper_listener_host }}" + port: "{{ cifmw_skupper_listener_port }}" + type: tcp + tlsCredentials: "{{ cifmw_skupper_listener_cert_secret }}" + + - name: Wait for Skupper Listener to be configured + # A Listener shows "Ready" only after its matching Connector activates the + # link. Waiting for "Configured" is sufficient here; both sides will + # transition to "Ready" once the application network is fully established. + when: + - cifmw_skupper_listener_enabled | bool + - _existing_listener.resources | length == 0 + ignore_errors: "{{ cifmw_skupper_listener_ignore_wait_errors | bool }}" + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Listener + name: "{{ cifmw_skupper_listener_name }}" + namespace: "{{ cifmw_skupper_listener_namespace }}" + register: _listener + retries: 30 + delay: 10 + until: + - _listener.resources | length > 0 + - _listener.resources[0].status is defined + - _listener.resources[0].status.conditions is defined + - _listener.resources[0].status.conditions | + selectattr('type', 'equalto', 'Configured') | + selectattr('status', 'equalto', 'True') | list | length > 0 diff --git a/hooks/playbooks/skmo/skupper-sites.yaml b/hooks/playbooks/skmo/skupper-sites.yaml new file mode 100644 index 000000000..d6358adf2 --- /dev/null +++ b/hooks/playbooks/skmo/skupper-sites.yaml @@ -0,0 +1,189 @@ +--- +# Create Skupper Sites in the central and leaf namespaces and link them. +# +# The central site is created with link access enabled so that it can issue +# tokens. The leaf site connects to the central site using an AccessToken +# derived from an AccessGrant, which avoids a dependency on the skupper CLI. +# +# Variables: +# cifmw_skupper_central_namespace (default: openstack) +# cifmw_skupper_leaf_namespace (default: openstack2) +# cifmw_skupper_central_site_name (default: openstack) +# cifmw_skupper_leaf_site_name (default: openstack2) +# cifmw_skupper_link_access_type (default: route) +# route - OpenShift Route (default; works on OpenShift without +# MetalLB / external LB) +# loadbalancer - LoadBalancer Service (multi-cluster with MetalLB or +# cloud LB) +# nodeport - NodePort Service +- name: Create and link Skupper sites + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + vars: + cifmw_skupper_central_namespace: openstack + cifmw_skupper_leaf_namespace: openstack2 + cifmw_skupper_central_site_name: openstack + cifmw_skupper_leaf_site_name: openstack2 + cifmw_skupper_link_access_type: route + tasks: + - name: Create Skupper Site in central namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: skupper.io/v2alpha1 + kind: Site + metadata: + name: "{{ cifmw_skupper_central_site_name }}" + namespace: "{{ cifmw_skupper_central_namespace }}" + spec: + linkAccess: "{{ cifmw_skupper_link_access_type }}" + + - name: Create Skupper Site in leaf namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: skupper.io/v2alpha1 + kind: Site + metadata: + name: "{{ cifmw_skupper_leaf_site_name }}" + namespace: "{{ cifmw_skupper_leaf_namespace }}" + spec: + linkAccess: none + + - name: Wait for central Skupper Site to be ready + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Site + name: "{{ cifmw_skupper_central_site_name }}" + namespace: "{{ cifmw_skupper_central_namespace }}" + register: _central_site + retries: 30 + delay: 10 + until: + - _central_site.resources | length > 0 + - _central_site.resources[0].status is defined + - _central_site.resources[0].status.conditions is defined + - _central_site.resources[0].status.conditions | + selectattr('type', 'equalto', 'Ready') | + selectattr('status', 'equalto', 'True') | list | length > 0 + + - name: Wait for leaf Skupper Site to be ready + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Site + name: "{{ cifmw_skupper_leaf_site_name }}" + namespace: "{{ cifmw_skupper_leaf_namespace }}" + register: _leaf_site + retries: 30 + delay: 10 + until: + - _leaf_site.resources | length > 0 + - _leaf_site.resources[0].status is defined + - _leaf_site.resources[0].status.conditions is defined + - _leaf_site.resources[0].status.conditions | + selectattr('type', 'equalto', 'Ready') | + selectattr('status', 'equalto', 'True') | list | length > 0 + + - name: Check if AccessGrant for leaf already exists + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: AccessGrant + name: "link-to-{{ cifmw_skupper_leaf_site_name }}" + namespace: "{{ cifmw_skupper_central_namespace }}" + register: _existing_grant + + - name: Create AccessGrant in central namespace for leaf to redeem + when: _existing_grant.resources | length == 0 + kubernetes.core.k8s: + state: present + definition: + apiVersion: skupper.io/v2alpha1 + kind: AccessGrant + metadata: + name: "link-to-{{ cifmw_skupper_leaf_site_name }}" + namespace: "{{ cifmw_skupper_central_namespace }}" + spec: + redemptionsAllowed: 1 + expirationWindow: 15m + + - name: Wait for AccessGrant to be ready + # In Skupper v2 the grant url/ca/code are stored in the AccessGrant status, + # not in a separate Kubernetes Secret. + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: AccessGrant + name: "link-to-{{ cifmw_skupper_leaf_site_name }}" + namespace: "{{ cifmw_skupper_central_namespace }}" + register: _access_grant + retries: 30 + delay: 10 + until: + - _access_grant.resources | length > 0 + - _access_grant.resources[0].status is defined + - _access_grant.resources[0].status.url is defined + - _access_grant.resources[0].status.ca is defined + - _access_grant.resources[0].status.code is defined + - _access_grant.resources[0].status.conditions is defined + - _access_grant.resources[0].status.conditions | + selectattr('type', 'equalto', 'Ready') | + selectattr('status', 'equalto', 'True') | list | length > 0 + + - name: Check if Link from leaf to central already exists + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Link + namespace: "{{ cifmw_skupper_leaf_namespace }}" + register: _existing_links + + - name: Check if a previous AccessToken exists in leaf namespace + when: _existing_links.resources | length == 0 + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: AccessToken + name: "link-to-{{ cifmw_skupper_central_site_name }}" + namespace: "{{ cifmw_skupper_leaf_namespace }}" + register: _existing_token + + - name: Delete stale AccessToken so the new grant can be redeemed + # An already-redeemed AccessToken cannot be redeemed again. Delete it so + # the fresh AccessGrant credentials can establish a new Link. + when: + - _existing_links.resources | length == 0 + - _existing_token.resources | length > 0 + kubernetes.core.k8s: + state: absent + api_version: skupper.io/v2alpha1 + kind: AccessToken + name: "link-to-{{ cifmw_skupper_central_site_name }}" + namespace: "{{ cifmw_skupper_leaf_namespace }}" + + - name: Create AccessToken in leaf namespace to establish link to central + when: _existing_links.resources | length == 0 + kubernetes.core.k8s: + state: present + definition: + apiVersion: skupper.io/v2alpha1 + kind: AccessToken + metadata: + name: "link-to-{{ cifmw_skupper_central_site_name }}" + namespace: "{{ cifmw_skupper_leaf_namespace }}" + spec: + url: "{{ _access_grant.resources[0].status.url }}" + ca: "{{ _access_grant.resources[0].status.ca }}" + code: "{{ _access_grant.resources[0].status.code }}" + + - name: Wait for Link in leaf namespace to be ready + kubernetes.core.k8s_info: + api_version: skupper.io/v2alpha1 + kind: Link + namespace: "{{ cifmw_skupper_leaf_namespace }}" + register: _leaf_link + retries: 30 + delay: 10 + until: + - _leaf_link.resources | length > 0 + - _leaf_link.resources[0].status is defined + - _leaf_link.resources[0].status.conditions is defined + - _leaf_link.resources[0].status.conditions | + selectattr('type', 'equalto', 'Ready') | + selectattr('status', 'equalto', 'True') | list | length > 0 diff --git a/hooks/playbooks/skmo/update-central-ca-bundle.yaml b/hooks/playbooks/skmo/update-central-ca-bundle.yaml index 784ed581a..8a1886dfb 100644 --- a/hooks/playbooks/skmo/update-central-ca-bundle.yaml +++ b/hooks/playbooks/skmo/update-central-ca-bundle.yaml @@ -1,6 +1,6 @@ --- - name: Update central CA bundle with leaf region CAs and wait for reconciliation - hosts: localhost + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" gather_facts: false vars: central_namespace: openstack