From 484fb79a000c5e3d63c90f864f785001cde719a3 Mon Sep 17 00:00:00 2001 From: Vito Castellano Date: Sun, 26 Apr 2026 23:30:37 +0200 Subject: [PATCH] feat(test_operator): add parallel_group support for concurrent tempest stages Add native parallel_group support to the test_operator Ansible role, enabling multiple Tempest CRs to execute simultaneously by leveraging the existing spec.parallel field in the test-operator CRD. Stages sharing the same parallel_group name are collected, built as individual Tempest CRs with spec.parallel: true, and applied concurrently. The first CR is applied ahead with a configurable wait (cifmw_test_operator_parallel_resource_wait) to allow discover-tempest-config --create to provision shared OpenStack resources before subsequent CRs start. Stages without parallel_group continue to execute sequentially, preserving full backward compatibility. After all parallel pods complete, logs are collected from each CR's PVCs via temporary pods, and all parallel resources are cleaned up automatically. Signed-off-by: Vito Castellano --- roles/test_operator/defaults/main.yml | 17 ++ .../test_operator/tasks/build-parallel-cr.yml | 99 +++++++++ .../test_operator/tasks/cleanup-parallel.yml | 50 +++++ .../tasks/collect-parallel-logs.yml | 158 +++++++++++++ roles/test_operator/tasks/dispatch-stage.yml | 62 ++++++ roles/test_operator/tasks/load-stage-vars.yml | 71 ++++++ roles/test_operator/tasks/main.yml | 9 +- roles/test_operator/tasks/parallel_stages.yml | 209 ++++++++++++++++++ roles/test_operator/tasks/stages.yml | 28 +-- 9 files changed, 675 insertions(+), 28 deletions(-) create mode 100644 roles/test_operator/tasks/build-parallel-cr.yml create mode 100644 roles/test_operator/tasks/cleanup-parallel.yml create mode 100644 roles/test_operator/tasks/collect-parallel-logs.yml create mode 100644 roles/test_operator/tasks/dispatch-stage.yml create mode 100644 roles/test_operator/tasks/load-stage-vars.yml create mode 100644 roles/test_operator/tasks/parallel_stages.yml diff --git a/roles/test_operator/defaults/main.yml b/roles/test_operator/defaults/main.yml index 08d131d0d..7ad65ea6a 100644 --- a/roles/test_operator/defaults/main.yml +++ b/roles/test_operator/defaults/main.yml @@ -87,6 +87,23 @@ cifmw_test_operator_tempest_resources: requests: {} limits: {} +# Section 1b: parallel execution parameters +# Used when stages share a parallel_group value. +# Seconds to wait after the first CR is applied, allowing shared +# resources (images, flavors) to be created before remaining CRs start. +cifmw_test_operator_parallel_resource_wait: 120 +# Maximum seconds to wait for all parallel pods to complete. +cifmw_test_operator_parallel_timeout: 14400 +# Resource requests/limits applied to each pod in a parallel group. +# Smaller than single-stage defaults so multiple pods fit on one node. +cifmw_test_operator_parallel_resources: + requests: + cpu: "500m" + memory: "512Mi" + limits: + cpu: "2" + memory: "2Gi" + # Enabling SRBAC by default, in jobs where this does not make sense should be turned off explicitly # # auth.tempest_roles is set to an empty value because otherwise diff --git a/roles/test_operator/tasks/build-parallel-cr.yml b/roles/test_operator/tasks/build-parallel-cr.yml new file mode 100644 index 000000000..ba08cff04 --- /dev/null +++ b/roles/test_operator/tasks/build-parallel-cr.yml @@ -0,0 +1,99 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Build a single Tempest CR for a parallel stage. +# +# This reuses the standard tempest-tests.yml logic (include/exclude lists, +# SSH key secret, controller IP injection, workflow overrides, etc.) to +# construct the CR, then applies parallel-specific modifications. +# +# Expected input: +# _pstage: stage dict (name, type, test_vars, etc.) +# ansible_loop.index0: position in the parallel group (0-based) +# +# Side effect: +# Appends the built CR to _parallel_cr_list fact. + +- name: "Build parallel CR for stage: {{ _pstage.name }}" + ansible.builtin.debug: + msg: "Building CR {{ ansible_loop.index0 + 1 }}/{{ ansible_loop.length }}: {{ _pstage.name }}" + +- name: "Load stage variables for {{ _pstage.name }}" + vars: + _stage_vars: "{{ _pstage }}" + ansible.builtin.include_tasks: load-stage-vars.yml + +- name: "Set CR base from tempest config template for {{ _pstage.name }}" + vars: + _stage_vars: "{{ _pstage }}" + ansible.builtin.set_fact: + test_operator_cr: "{{ stage_vars_dict.cifmw_test_operator_tempest_config }}" + +- name: "Clear inherited workflow for parallel stage {{ _pstage.name }}" + ansible.builtin.set_fact: + stage_vars_dict: >- + {{ + stage_vars_dict | combine({ + 'cifmw_test_operator_tempest_workflow': [] + }) + }} + +- name: "Apply tempest-specific configuration for {{ _pstage.name }}" + vars: + _stage_vars: "{{ _pstage }}" + run_test_fw: tempest + test_operator_instance_name: >- + {{ stage_vars_dict.cifmw_test_operator_tempest_name }}-{{ _pstage.name }} + test_operator_workflow: [] + ansible.builtin.include_tasks: tempest-tests.yml + +- name: "Apply parallel-specific overrides for {{ _pstage.name }}" + ansible.builtin.set_fact: + test_operator_cr: >- + {{ + test_operator_cr | combine({ + 'spec': { + 'parallel': true, + 'workflow': [], + 'resources': cifmw_test_operator_parallel_resources | + default(test_operator_cr.spec.resources | default({})), + 'tempestconfRun': + test_operator_cr.spec.tempestconfRun | default({}) | combine({ + 'create': (ansible_loop.index0 == 0) + }), + 'tempestRun': + test_operator_cr.spec.tempestRun | default({}) | combine({ + 'extraImages': ( + test_operator_cr.spec.tempestRun.extraImages | default([]) + if ansible_loop.index0 == 0 + else []) + }), + 'rerunFailedTests': + _pstage.rerunFailedTests | default( + stage_vars_dict.cifmw_test_operator_tempest_rerun_failed_tests | + default(false)) | bool, + 'rerunOverrideStatus': + _pstage.rerunOverrideStatus | default( + stage_vars_dict.cifmw_test_operator_tempest_rerun_override_status | + default(false)) | bool, + } + }, recursive=true) + }} + +- name: "Append CR to parallel list: {{ _pstage.name }}" + ansible.builtin.set_fact: + _parallel_cr_list: >- + {{ _parallel_cr_list | default([]) + [test_operator_cr] }} diff --git a/roles/test_operator/tasks/cleanup-parallel.yml b/roles/test_operator/tasks/cleanup-parallel.yml new file mode 100644 index 000000000..6bb27cf64 --- /dev/null +++ b/roles/test_operator/tasks/cleanup-parallel.yml @@ -0,0 +1,50 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Clean up resources created by parallel Tempest CRs. +# +# Expected input: +# _parallel_cr_list: list of Tempest CR definition dicts + +- name: Delete parallel Tempest CRs + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: absent + definition: "{{ item }}" + wait: true + wait_timeout: 600 + loop: "{{ _parallel_cr_list }}" + loop_control: + label: "{{ item.metadata.name }}" + +- name: Delete parallel log pods + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: absent + api_version: v1 + kind: Pod + name: "test-operator-logs-pod-tempest-{{ item.metadata.name }}" + namespace: "{{ item.metadata.namespace }}" + wait: true + wait_timeout: 600 + when: cifmw_test_operator_delete_logs_pod | bool or cifmw_test_operator_cleanup | bool + loop: "{{ _parallel_cr_list }}" + loop_control: + label: "{{ item.metadata.name }}" diff --git a/roles/test_operator/tasks/collect-parallel-logs.yml b/roles/test_operator/tasks/collect-parallel-logs.yml new file mode 100644 index 000000000..d2567bd1a --- /dev/null +++ b/roles/test_operator/tasks/collect-parallel-logs.yml @@ -0,0 +1,158 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Collect logs from a single parallel Tempest CR's PVCs. +# +# Reuses the same PVC-based log collection approach as collect-logs.yaml +# but scoped to a specific parallel CR instance. +# +# Expected input: +# _pcr: the Tempest CR definition dict (with .metadata.name, .metadata.namespace) + +- name: "Reset volumes for {{ _pcr.metadata.name }}" + ansible.builtin.set_fact: + _test_operator_volumes: [] + _test_operator_volume_mounts: [] + +- name: "Get PVCs for {{ _pcr.metadata.name }}" + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + namespace: "{{ _pcr.metadata.namespace }}" + kind: PersistentVolumeClaim + label_selectors: + - "instanceName={{ _pcr.metadata.name }}" + register: _pcr_pvcs + +- name: "Set up volume mounts for {{ _pcr.metadata.name }}" + ansible.builtin.set_fact: + _test_operator_volume_mounts: >- + {{ + (_test_operator_volume_mounts | default([])) + [{ + 'name': 'logs-volume-' ~ index, + 'mountPath': '/mnt/logs-' + _pcr.metadata.name + '-step-' ~ index + }] + }} + _test_operator_volumes: >- + {{ + (_test_operator_volumes | default([])) + [{ + 'name': 'logs-volume-' ~ index, + 'persistentVolumeClaim': { + 'claimName': pvc.metadata.name + } + }] + }} + loop: "{{ _pcr_pvcs.resources }}" + loop_control: + loop_var: pvc + index_var: index + +- name: "Create log pod definition for {{ _pcr.metadata.name }}" + when: _pcr_pvcs.resources | length > 0 + block: + - name: Set log pod fact + vars: + run_test_fw: tempest + test_operator_instance_name: "{{ _pcr.metadata.name }}" + ansible.builtin.set_fact: + _test_operator_log_pod: "{{ cifmw_test_operator_log_pod_definition }}" + + - name: "Write log pod definition for {{ _pcr.metadata.name }}" + ansible.builtin.copy: + content: "{{ _test_operator_log_pod | to_nice_yaml }}" + dest: "{{ cifmw_test_operator_crs_path }}/{{ _pcr.metadata.name }}-log-pod.yaml" + mode: '0644' + + - name: "Start log pod for {{ _pcr.metadata.name }}" + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + wait: true + src: "{{ cifmw_test_operator_crs_path }}/{{ _pcr.metadata.name }}-log-pod.yaml" + + - name: "Ensure log pod is Running for {{ _pcr.metadata.name }}" + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + namespace: "{{ _pcr.metadata.namespace }}" + kind: Pod + name: "test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}" + wait: true + register: _pcr_logs_pod + until: _pcr_logs_pod.resources[0].status.phase == "Running" + delay: 10 + retries: 20 + + - name: "Copy logs from {{ _pcr.metadata.name }}" + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + _pod_path: "mnt/logs-{{ _pcr.metadata.name }}-step-{{ index }}" + ansible.builtin.shell: >- + oc cp -n {{ _pcr.metadata.namespace }} + test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}:{{ _pod_path }} + {{ cifmw_test_operator_artifacts_basedir }} + loop: "{{ _pcr_pvcs.resources }}" + loop_control: + index_var: index + + - name: "Find subunit files for {{ _pcr.metadata.name }}" + failed_when: false + ansible.builtin.find: + paths: "{{ cifmw_test_operator_artifacts_basedir }}" + patterns: "*.subunit" + recurse: true + register: _pcr_subunit_files + + - name: Install subunit and stestr packages + become: true + when: _pcr_subunit_files.files | default([]) | length > 0 + failed_when: false + ansible.builtin.dnf: + name: + - python3-subunit + - python3-stestr + state: present + + - name: "Generate HTML reports for {{ _pcr.metadata.name }}" + when: _pcr_subunit_files.files | default([]) | length > 0 + failed_when: false + ansible.builtin.command: + cmd: >- + python3 {{ role_path }}/files/subunit-to-html.py + {{ item.path }} + {{ item.path | regex_replace('\.subunit$', '-viz.html') }} + loop: "{{ _pcr_subunit_files.files }}" + loop_control: + label: "{{ item.path | basename }}" + + - name: "Delete log pod for {{ _pcr.metadata.name }}" + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: absent + api_version: v1 + kind: Pod + name: "test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}" + namespace: "{{ _pcr.metadata.namespace }}" + wait: true + wait_timeout: 120 diff --git a/roles/test_operator/tasks/dispatch-stage.yml b/roles/test_operator/tasks/dispatch-stage.yml new file mode 100644 index 000000000..a77c4fa11 --- /dev/null +++ b/roles/test_operator/tasks/dispatch-stage.yml @@ -0,0 +1,62 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Dispatch a single stage to either sequential or parallel execution. +# +# Stages with parallel_group are collected and executed together the first +# time the group is encountered. Subsequent stages in the same group are +# skipped (already processed). +# +# Stages without parallel_group are executed sequentially as before. +# +# Expected input: +# _stage_vars: the current stage dict from the main loop + +- name: "Process parallel group: {{ _stage_vars.parallel_group }}" + when: + - _stage_vars.parallel_group is defined + - _stage_vars.parallel_group not in (_processed_parallel_groups | default([])) + block: + - name: "Collect all stages in parallel group: {{ _stage_vars.parallel_group }}" + ansible.builtin.set_fact: + _parallel_stages: >- + {{ + cifmw_test_operator_stages | + selectattr('parallel_group', 'defined') | + selectattr('parallel_group', 'equalto', _stage_vars.parallel_group) | + list + }} + + - name: Execute parallel stages + ansible.builtin.include_tasks: parallel_stages.yml + + - name: Mark parallel group as processed + ansible.builtin.set_fact: + _processed_parallel_groups: >- + {{ (_processed_parallel_groups | default([])) + [_stage_vars.parallel_group] }} + +- name: "Skip already-processed parallel stage: {{ _stage_vars.name }}" + when: + - _stage_vars.parallel_group is defined + - _stage_vars.parallel_group in (_processed_parallel_groups | default([])) + ansible.builtin.debug: + msg: >- + Skipping {{ _stage_vars.name }} - parallel group + {{ _stage_vars.parallel_group }} already processed. + +- name: "Process sequential stage: {{ _stage_vars.name }}" + when: _stage_vars.parallel_group is not defined + ansible.builtin.include_tasks: stages.yml diff --git a/roles/test_operator/tasks/load-stage-vars.yml b/roles/test_operator/tasks/load-stage-vars.yml new file mode 100644 index 000000000..1d7cc5c3f --- /dev/null +++ b/roles/test_operator/tasks/load-stage-vars.yml @@ -0,0 +1,71 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Load and merge stage-level variable overrides into stage_vars_dict. +# +# Expected input: +# _stage_vars: dict with at least .type and optionally +# .test_vars_file and .test_vars +# +# Output: +# stage_vars_dict: merged cifmw_test_operator_* variables for this stage. + +- name: Set empty stage vars dict + ansible.builtin.set_fact: + stage_vars_dict: {} + +- name: Include stage var file + ansible.builtin.include_vars: + file: "{{ _stage_vars.test_vars_file | default('/dev/null') }}" + name: _stage_var_file + +- name: Merge file vars and test vars + vars: + file_vars: "{{ _stage_var_file }}" + test_vars: "{{ _stage_vars.test_vars | default({}) }}" + ansible.builtin.set_fact: + _stage_test_vars: "{{ file_vars | combine(test_vars) }}" + +- name: Overwrite global_vars with stage_vars + ansible.builtin.set_fact: + stage_vars_dict: >- + {{ + stage_vars_dict | + combine({ + item.key: _stage_test_vars[item.key] | + default(lookup('vars', item.key, default=omit)) + }) + }} + with_dict: >- + {{ + vars | combine(_stage_test_vars) | + dict2items | + selectattr('key', 'match', '^cifmw_test_operator_') | + items2dict + }} + +- name: Override specific type config + vars: + _stage_config: 'cifmw_test_operator_{{ _stage_vars.type }}_config' + ansible.builtin.set_fact: + stage_vars_dict: >- + {{ + stage_vars_dict | + combine({ + _stage_config: _stage_test_vars[_stage_config] | + default(lookup('vars', _stage_config, default=omit)) + }) + }} diff --git a/roles/test_operator/tasks/main.yml b/roles/test_operator/tasks/main.yml index a83a3c181..3b7c1c9c3 100644 --- a/roles/test_operator/tasks/main.yml +++ b/roles/test_operator/tasks/main.yml @@ -163,9 +163,14 @@ args: chdir: /tmp/test-operator -- name: Call test stages loop +- name: Initialize parallel group tracking when: not cifmw_test_operator_dry_run | bool - ansible.builtin.include_tasks: stages.yml + ansible.builtin.set_fact: + _processed_parallel_groups: [] + +- name: Dispatch test stages + when: not cifmw_test_operator_dry_run | bool + ansible.builtin.include_tasks: dispatch-stage.yml loop: "{{ cifmw_test_operator_stages }}" loop_control: loop_var: _stage_vars diff --git a/roles/test_operator/tasks/parallel_stages.yml b/roles/test_operator/tasks/parallel_stages.yml new file mode 100644 index 000000000..8a478bfca --- /dev/null +++ b/roles/test_operator/tasks/parallel_stages.yml @@ -0,0 +1,209 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Execute a group of Tempest stages in parallel. +# +# Applies multiple independent Tempest CRs simultaneously, leveraging +# the test-operator's inter-CR parallelism (spec.parallel: true). +# The first CR creates shared resources (images, flavors) while the +# remaining CRs wait before starting, avoiding race conditions. +# +# Expected input: +# _parallel_stages: list of stage dicts sharing the same parallel_group. +# Each must have .name and .type == 'tempest'. +# +# The task: +# 1. Runs pre_test_stage_hooks for the first stage (if defined) +# 2. Builds a Tempest CR for each stage (reusing tempest-tests.yml) +# 3. Applies the first CR, pauses, then applies the rest +# 4. Waits for all pods to reach Succeeded or Failed +# 5. Collects logs from all pods +# 6. Records per-stage pass/fail results +# 7. Runs post_test_stage_hooks for the last stage (if defined) + +- name: "Run parallel group: {{ _parallel_stages | map(attribute='name') | join(', ') }}" + ansible.builtin.debug: + msg: >- + Running {{ _parallel_stages | length }} stages in parallel: + {{ _parallel_stages | map(attribute='name') | list | join(', ') }} + +- name: "Call pre stage hooks for parallel group" + when: + - not cifmw_test_operator_dry_run | bool + - _parallel_stages[0].pre_test_stage_hooks is defined + vars: + step: pre_test_hooks + hooks: "{{ _parallel_stages[0].pre_test_stage_hooks }}" + ansible.builtin.import_role: + name: run_hook + +- name: Initialize parallel CR list + ansible.builtin.set_fact: + _parallel_cr_list: [] + +- name: Build Tempest CRs for parallel group + ansible.builtin.include_tasks: build-parallel-cr.yml + loop: "{{ _parallel_stages }}" + loop_control: + loop_var: _pstage + extended: true + +- name: Log parallel CR names + ansible.builtin.debug: + msg: >- + Built {{ _parallel_cr_list | length }} CRs: + {{ _parallel_cr_list | map(attribute='metadata.name') | list | join(', ') }} + +- name: Parallel apply and wait block + when: not cifmw_test_operator_dry_run | bool + block: + - name: Make sure test-operator CR directory exists + ansible.builtin.file: + path: "{{ cifmw_test_operator_crs_path }}" + state: directory + mode: '0755' + + - name: Write all parallel CRs to disk + ansible.builtin.copy: + content: "{{ item | to_nice_yaml }}" + dest: "{{ cifmw_test_operator_crs_path }}/{{ item.metadata.name }}.yaml" + mode: '0644' + loop: "{{ _parallel_cr_list }}" + loop_control: + label: "{{ item.metadata.name }}" + + - name: "Apply first Tempest CR (creates shared images/flavors)" + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + definition: "{{ _parallel_cr_list[0] }}" + + - name: >- + Wait {{ cifmw_test_operator_parallel_resource_wait }}s + for first pod to create shared resources + ansible.builtin.pause: + seconds: "{{ cifmw_test_operator_parallel_resource_wait | int }}" + + - name: Apply remaining Tempest CRs + when: _parallel_cr_list | length > 1 + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + definition: "{{ item }}" + loop: "{{ _parallel_cr_list[1:] }}" + loop_control: + label: "{{ item.metadata.name }}" + + - name: "Wait for all {{ _parallel_cr_list | length }} Tempest pods to complete" + vars: + _parallel_timeout: >- + {{ cifmw_test_operator_parallel_timeout | default(cifmw_test_operator_timeout) }} + _parallel_prefix: >- + {{ _parallel_cr_list[0].metadata.name | regex_replace('-[^-]+$', '') }} + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + namespace: "{{ _parallel_cr_list[0].metadata.namespace }}" + kind: Pod + label_selectors: + - "service=tempest" + register: _parallel_pods + retries: "{{ ((_parallel_timeout | int) / 30) | int }}" + delay: 30 + until: >- + _parallel_pods.resources is defined and + (_parallel_pods.resources | + selectattr('metadata.labels.instanceName', 'defined') | + selectattr('metadata.labels.instanceName', 'in', + _parallel_cr_list | map(attribute='metadata.name') | list) | + selectattr('status.phase', 'defined') | + selectattr('status.phase', 'in', ['Succeeded', 'Failed']) | + list | length) >= (_parallel_cr_list | length) + ignore_errors: true + + - name: Check whether timed out + ansible.builtin.set_fact: + _parallel_timed_out: >- + {{ + _parallel_pods.attempts | default(0) >= + ((cifmw_test_operator_parallel_timeout | + default(cifmw_test_operator_timeout)) | int / 30) | int + }} + + - name: Collect logs for each parallel stage + when: not _parallel_timed_out + ansible.builtin.include_tasks: collect-parallel-logs.yml + loop: "{{ _parallel_cr_list }}" + loop_control: + loop_var: _pcr + label: "{{ _pcr.metadata.name }}" + + - name: Get results from all parallel test pods + vars: + _completed_pods: >- + {{ (_parallel_pods.resources | default([])) | + selectattr('metadata.labels.instanceName', 'defined') | + selectattr('metadata.labels.instanceName', 'in', + _parallel_cr_list | map(attribute='metadata.name') | list) | + list }} + _succeeded_pods: >- + {{ _completed_pods | + selectattr('status.phase', 'equalto', 'Succeeded') | list }} + _failed_pods: >- + {{ _completed_pods | + selectattr('status.phase', 'equalto', 'Failed') | list }} + block: + - name: Print parallel results summary + ansible.builtin.debug: + msg: | + === Parallel Tempest Results === + Total pods: {{ _completed_pods | length }} + Succeeded: {{ _succeeded_pods | length }} + Failed: {{ _failed_pods | length }} + Failed names: {{ _failed_pods | map(attribute='metadata.name') | list | join(', ') }} + + - name: Save per-stage results + ansible.builtin.set_fact: + test_operator_results: >- + {{ + test_operator_results | default({}) | + combine({ + 'tempest-' + item.metadata.labels.instanceName: + (item.status.phase == 'Succeeded') + }) + }} + loop: "{{ _completed_pods }}" + loop_control: + label: "{{ item.metadata.labels.instanceName }}" + + - name: Delete test resources + when: cifmw_test_operator_cleanup | bool + ansible.builtin.include_tasks: cleanup-parallel.yml + +- name: "Call post stage hooks for parallel group" + when: + - not cifmw_test_operator_dry_run | bool + - _parallel_stages[-1].post_test_stage_hooks is defined + vars: + step: post_test_hooks + hooks: "{{ _parallel_stages[-1].post_test_stage_hooks }}" + ansible.builtin.import_role: + name: run_hook diff --git a/roles/test_operator/tasks/stages.yml b/roles/test_operator/tasks/stages.yml index 675018677..8eba568ef 100644 --- a/roles/test_operator/tasks/stages.yml +++ b/roles/test_operator/tasks/stages.yml @@ -13,32 +13,8 @@ ansible.builtin.import_role: name: run_hook -- name: Set empty stage vars dict - ansible.builtin.set_fact: - stage_vars_dict: {} - -- name: Include stage var file - ansible.builtin.include_vars: - file: "{{ _stage_vars.test_vars_file | default('/dev/null') }}" - name: _stage_var_file - -- name: Merge file vars and test vars - vars: - file_vars: "{{ _stage_var_file }}" - test_vars: "{{ _stage_vars.test_vars | default({}) }}" - ansible.builtin.set_fact: - _stage_test_vars: "{{ file_vars | combine(test_vars) }}" - -- name: Overwrite global_vars with stage_vars - ansible.builtin.set_fact: - stage_vars_dict: "{{ stage_vars_dict | combine({item.key: _stage_test_vars[item.key] | default(lookup('vars', item.key, default=omit)) }) }}" - with_dict: "{{ vars | combine(_stage_test_vars) | dict2items | selectattr('key', 'match', '^cifmw_test_operator_') | items2dict }}" - -- name: Override specific type config - vars: - _stage_config: 'cifmw_test_operator_{{ _stage_vars.type }}_config' - ansible.builtin.set_fact: - stage_vars_dict: "{{ stage_vars_dict | combine({_stage_config: _stage_test_vars[_stage_config] | default(lookup('vars', _stage_config, default=omit)) }) }}" +- name: Load stage variables + ansible.builtin.include_tasks: load-stage-vars.yml - name: "Call runner {{ _stage_vars.type }}" ansible.builtin.include_tasks: "runners/{{ _stage_vars.type }}_runner.yml"