diff --git a/roles/test_operator/defaults/main.yml b/roles/test_operator/defaults/main.yml index 08d131d0d..7ad65ea6a 100644 --- a/roles/test_operator/defaults/main.yml +++ b/roles/test_operator/defaults/main.yml @@ -87,6 +87,23 @@ cifmw_test_operator_tempest_resources: requests: {} limits: {} +# Section 1b: parallel execution parameters +# Used when stages share a parallel_group value. +# Seconds to wait after the first CR is applied, allowing shared +# resources (images, flavors) to be created before remaining CRs start. +cifmw_test_operator_parallel_resource_wait: 120 +# Maximum seconds to wait for all parallel pods to complete. +cifmw_test_operator_parallel_timeout: 14400 +# Resource requests/limits applied to each pod in a parallel group. +# Smaller than single-stage defaults so multiple pods fit on one node. +cifmw_test_operator_parallel_resources: + requests: + cpu: "500m" + memory: "512Mi" + limits: + cpu: "2" + memory: "2Gi" + # Enabling SRBAC by default, in jobs where this does not make sense should be turned off explicitly # # auth.tempest_roles is set to an empty value because otherwise diff --git a/roles/test_operator/tasks/build-parallel-cr.yml b/roles/test_operator/tasks/build-parallel-cr.yml new file mode 100644 index 000000000..ba08cff04 --- /dev/null +++ b/roles/test_operator/tasks/build-parallel-cr.yml @@ -0,0 +1,99 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Build a single Tempest CR for a parallel stage. +# +# This reuses the standard tempest-tests.yml logic (include/exclude lists, +# SSH key secret, controller IP injection, workflow overrides, etc.) to +# construct the CR, then applies parallel-specific modifications. +# +# Expected input: +# _pstage: stage dict (name, type, test_vars, etc.) +# ansible_loop.index0: position in the parallel group (0-based) +# +# Side effect: +# Appends the built CR to _parallel_cr_list fact. + +- name: "Build parallel CR for stage: {{ _pstage.name }}" + ansible.builtin.debug: + msg: "Building CR {{ ansible_loop.index0 + 1 }}/{{ ansible_loop.length }}: {{ _pstage.name }}" + +- name: "Load stage variables for {{ _pstage.name }}" + vars: + _stage_vars: "{{ _pstage }}" + ansible.builtin.include_tasks: load-stage-vars.yml + +- name: "Set CR base from tempest config template for {{ _pstage.name }}" + vars: + _stage_vars: "{{ _pstage }}" + ansible.builtin.set_fact: + test_operator_cr: "{{ stage_vars_dict.cifmw_test_operator_tempest_config }}" + +- name: "Clear inherited workflow for parallel stage {{ _pstage.name }}" + ansible.builtin.set_fact: + stage_vars_dict: >- + {{ + stage_vars_dict | combine({ + 'cifmw_test_operator_tempest_workflow': [] + }) + }} + +- name: "Apply tempest-specific configuration for {{ _pstage.name }}" + vars: + _stage_vars: "{{ _pstage }}" + run_test_fw: tempest + test_operator_instance_name: >- + {{ stage_vars_dict.cifmw_test_operator_tempest_name }}-{{ _pstage.name }} + test_operator_workflow: [] + ansible.builtin.include_tasks: tempest-tests.yml + +- name: "Apply parallel-specific overrides for {{ _pstage.name }}" + ansible.builtin.set_fact: + test_operator_cr: >- + {{ + test_operator_cr | combine({ + 'spec': { + 'parallel': true, + 'workflow': [], + 'resources': cifmw_test_operator_parallel_resources | + default(test_operator_cr.spec.resources | default({})), + 'tempestconfRun': + test_operator_cr.spec.tempestconfRun | default({}) | combine({ + 'create': (ansible_loop.index0 == 0) + }), + 'tempestRun': + test_operator_cr.spec.tempestRun | default({}) | combine({ + 'extraImages': ( + test_operator_cr.spec.tempestRun.extraImages | default([]) + if ansible_loop.index0 == 0 + else []) + }), + 'rerunFailedTests': + _pstage.rerunFailedTests | default( + stage_vars_dict.cifmw_test_operator_tempest_rerun_failed_tests | + default(false)) | bool, + 'rerunOverrideStatus': + _pstage.rerunOverrideStatus | default( + stage_vars_dict.cifmw_test_operator_tempest_rerun_override_status | + default(false)) | bool, + } + }, recursive=true) + }} + +- name: "Append CR to parallel list: {{ _pstage.name }}" + ansible.builtin.set_fact: + _parallel_cr_list: >- + {{ _parallel_cr_list | default([]) + [test_operator_cr] }} diff --git a/roles/test_operator/tasks/cleanup-parallel.yml b/roles/test_operator/tasks/cleanup-parallel.yml new file mode 100644 index 000000000..6bb27cf64 --- /dev/null +++ b/roles/test_operator/tasks/cleanup-parallel.yml @@ -0,0 +1,50 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Clean up resources created by parallel Tempest CRs. +# +# Expected input: +# _parallel_cr_list: list of Tempest CR definition dicts + +- name: Delete parallel Tempest CRs + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: absent + definition: "{{ item }}" + wait: true + wait_timeout: 600 + loop: "{{ _parallel_cr_list }}" + loop_control: + label: "{{ item.metadata.name }}" + +- name: Delete parallel log pods + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: absent + api_version: v1 + kind: Pod + name: "test-operator-logs-pod-tempest-{{ item.metadata.name }}" + namespace: "{{ item.metadata.namespace }}" + wait: true + wait_timeout: 600 + when: cifmw_test_operator_delete_logs_pod | bool or cifmw_test_operator_cleanup | bool + loop: "{{ _parallel_cr_list }}" + loop_control: + label: "{{ item.metadata.name }}" diff --git a/roles/test_operator/tasks/collect-parallel-logs.yml b/roles/test_operator/tasks/collect-parallel-logs.yml new file mode 100644 index 000000000..d2567bd1a --- /dev/null +++ b/roles/test_operator/tasks/collect-parallel-logs.yml @@ -0,0 +1,158 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Collect logs from a single parallel Tempest CR's PVCs. +# +# Reuses the same PVC-based log collection approach as collect-logs.yaml +# but scoped to a specific parallel CR instance. +# +# Expected input: +# _pcr: the Tempest CR definition dict (with .metadata.name, .metadata.namespace) + +- name: "Reset volumes for {{ _pcr.metadata.name }}" + ansible.builtin.set_fact: + _test_operator_volumes: [] + _test_operator_volume_mounts: [] + +- name: "Get PVCs for {{ _pcr.metadata.name }}" + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + namespace: "{{ _pcr.metadata.namespace }}" + kind: PersistentVolumeClaim + label_selectors: + - "instanceName={{ _pcr.metadata.name }}" + register: _pcr_pvcs + +- name: "Set up volume mounts for {{ _pcr.metadata.name }}" + ansible.builtin.set_fact: + _test_operator_volume_mounts: >- + {{ + (_test_operator_volume_mounts | default([])) + [{ + 'name': 'logs-volume-' ~ index, + 'mountPath': '/mnt/logs-' + _pcr.metadata.name + '-step-' ~ index + }] + }} + _test_operator_volumes: >- + {{ + (_test_operator_volumes | default([])) + [{ + 'name': 'logs-volume-' ~ index, + 'persistentVolumeClaim': { + 'claimName': pvc.metadata.name + } + }] + }} + loop: "{{ _pcr_pvcs.resources }}" + loop_control: + loop_var: pvc + index_var: index + +- name: "Create log pod definition for {{ _pcr.metadata.name }}" + when: _pcr_pvcs.resources | length > 0 + block: + - name: Set log pod fact + vars: + run_test_fw: tempest + test_operator_instance_name: "{{ _pcr.metadata.name }}" + ansible.builtin.set_fact: + _test_operator_log_pod: "{{ cifmw_test_operator_log_pod_definition }}" + + - name: "Write log pod definition for {{ _pcr.metadata.name }}" + ansible.builtin.copy: + content: "{{ _test_operator_log_pod | to_nice_yaml }}" + dest: "{{ cifmw_test_operator_crs_path }}/{{ _pcr.metadata.name }}-log-pod.yaml" + mode: '0644' + + - name: "Start log pod for {{ _pcr.metadata.name }}" + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + wait: true + src: "{{ cifmw_test_operator_crs_path }}/{{ _pcr.metadata.name }}-log-pod.yaml" + + - name: "Ensure log pod is Running for {{ _pcr.metadata.name }}" + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + namespace: "{{ _pcr.metadata.namespace }}" + kind: Pod + name: "test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}" + wait: true + register: _pcr_logs_pod + until: _pcr_logs_pod.resources[0].status.phase == "Running" + delay: 10 + retries: 20 + + - name: "Copy logs from {{ _pcr.metadata.name }}" + environment: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + PATH: "{{ cifmw_path }}" + vars: + _pod_path: "mnt/logs-{{ _pcr.metadata.name }}-step-{{ index }}" + ansible.builtin.shell: >- + oc cp -n {{ _pcr.metadata.namespace }} + test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}:{{ _pod_path }} + {{ cifmw_test_operator_artifacts_basedir }} + loop: "{{ _pcr_pvcs.resources }}" + loop_control: + index_var: index + + - name: "Find subunit files for {{ _pcr.metadata.name }}" + failed_when: false + ansible.builtin.find: + paths: "{{ cifmw_test_operator_artifacts_basedir }}" + patterns: "*.subunit" + recurse: true + register: _pcr_subunit_files + + - name: Install subunit and stestr packages + become: true + when: _pcr_subunit_files.files | default([]) | length > 0 + failed_when: false + ansible.builtin.dnf: + name: + - python3-subunit + - python3-stestr + state: present + + - name: "Generate HTML reports for {{ _pcr.metadata.name }}" + when: _pcr_subunit_files.files | default([]) | length > 0 + failed_when: false + ansible.builtin.command: + cmd: >- + python3 {{ role_path }}/files/subunit-to-html.py + {{ item.path }} + {{ item.path | regex_replace('\.subunit$', '-viz.html') }} + loop: "{{ _pcr_subunit_files.files }}" + loop_control: + label: "{{ item.path | basename }}" + + - name: "Delete log pod for {{ _pcr.metadata.name }}" + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: absent + api_version: v1 + kind: Pod + name: "test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}" + namespace: "{{ _pcr.metadata.namespace }}" + wait: true + wait_timeout: 120 diff --git a/roles/test_operator/tasks/dispatch-stage.yml b/roles/test_operator/tasks/dispatch-stage.yml new file mode 100644 index 000000000..a77c4fa11 --- /dev/null +++ b/roles/test_operator/tasks/dispatch-stage.yml @@ -0,0 +1,62 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Dispatch a single stage to either sequential or parallel execution. +# +# Stages with parallel_group are collected and executed together the first +# time the group is encountered. Subsequent stages in the same group are +# skipped (already processed). +# +# Stages without parallel_group are executed sequentially as before. +# +# Expected input: +# _stage_vars: the current stage dict from the main loop + +- name: "Process parallel group: {{ _stage_vars.parallel_group }}" + when: + - _stage_vars.parallel_group is defined + - _stage_vars.parallel_group not in (_processed_parallel_groups | default([])) + block: + - name: "Collect all stages in parallel group: {{ _stage_vars.parallel_group }}" + ansible.builtin.set_fact: + _parallel_stages: >- + {{ + cifmw_test_operator_stages | + selectattr('parallel_group', 'defined') | + selectattr('parallel_group', 'equalto', _stage_vars.parallel_group) | + list + }} + + - name: Execute parallel stages + ansible.builtin.include_tasks: parallel_stages.yml + + - name: Mark parallel group as processed + ansible.builtin.set_fact: + _processed_parallel_groups: >- + {{ (_processed_parallel_groups | default([])) + [_stage_vars.parallel_group] }} + +- name: "Skip already-processed parallel stage: {{ _stage_vars.name }}" + when: + - _stage_vars.parallel_group is defined + - _stage_vars.parallel_group in (_processed_parallel_groups | default([])) + ansible.builtin.debug: + msg: >- + Skipping {{ _stage_vars.name }} - parallel group + {{ _stage_vars.parallel_group }} already processed. + +- name: "Process sequential stage: {{ _stage_vars.name }}" + when: _stage_vars.parallel_group is not defined + ansible.builtin.include_tasks: stages.yml diff --git a/roles/test_operator/tasks/load-stage-vars.yml b/roles/test_operator/tasks/load-stage-vars.yml new file mode 100644 index 000000000..1d7cc5c3f --- /dev/null +++ b/roles/test_operator/tasks/load-stage-vars.yml @@ -0,0 +1,71 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Load and merge stage-level variable overrides into stage_vars_dict. +# +# Expected input: +# _stage_vars: dict with at least .type and optionally +# .test_vars_file and .test_vars +# +# Output: +# stage_vars_dict: merged cifmw_test_operator_* variables for this stage. + +- name: Set empty stage vars dict + ansible.builtin.set_fact: + stage_vars_dict: {} + +- name: Include stage var file + ansible.builtin.include_vars: + file: "{{ _stage_vars.test_vars_file | default('/dev/null') }}" + name: _stage_var_file + +- name: Merge file vars and test vars + vars: + file_vars: "{{ _stage_var_file }}" + test_vars: "{{ _stage_vars.test_vars | default({}) }}" + ansible.builtin.set_fact: + _stage_test_vars: "{{ file_vars | combine(test_vars) }}" + +- name: Overwrite global_vars with stage_vars + ansible.builtin.set_fact: + stage_vars_dict: >- + {{ + stage_vars_dict | + combine({ + item.key: _stage_test_vars[item.key] | + default(lookup('vars', item.key, default=omit)) + }) + }} + with_dict: >- + {{ + vars | combine(_stage_test_vars) | + dict2items | + selectattr('key', 'match', '^cifmw_test_operator_') | + items2dict + }} + +- name: Override specific type config + vars: + _stage_config: 'cifmw_test_operator_{{ _stage_vars.type }}_config' + ansible.builtin.set_fact: + stage_vars_dict: >- + {{ + stage_vars_dict | + combine({ + _stage_config: _stage_test_vars[_stage_config] | + default(lookup('vars', _stage_config, default=omit)) + }) + }} diff --git a/roles/test_operator/tasks/main.yml b/roles/test_operator/tasks/main.yml index a83a3c181..3b7c1c9c3 100644 --- a/roles/test_operator/tasks/main.yml +++ b/roles/test_operator/tasks/main.yml @@ -163,9 +163,14 @@ args: chdir: /tmp/test-operator -- name: Call test stages loop +- name: Initialize parallel group tracking when: not cifmw_test_operator_dry_run | bool - ansible.builtin.include_tasks: stages.yml + ansible.builtin.set_fact: + _processed_parallel_groups: [] + +- name: Dispatch test stages + when: not cifmw_test_operator_dry_run | bool + ansible.builtin.include_tasks: dispatch-stage.yml loop: "{{ cifmw_test_operator_stages }}" loop_control: loop_var: _stage_vars diff --git a/roles/test_operator/tasks/parallel_stages.yml b/roles/test_operator/tasks/parallel_stages.yml new file mode 100644 index 000000000..8a478bfca --- /dev/null +++ b/roles/test_operator/tasks/parallel_stages.yml @@ -0,0 +1,209 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Execute a group of Tempest stages in parallel. +# +# Applies multiple independent Tempest CRs simultaneously, leveraging +# the test-operator's inter-CR parallelism (spec.parallel: true). +# The first CR creates shared resources (images, flavors) while the +# remaining CRs wait before starting, avoiding race conditions. +# +# Expected input: +# _parallel_stages: list of stage dicts sharing the same parallel_group. +# Each must have .name and .type == 'tempest'. +# +# The task: +# 1. Runs pre_test_stage_hooks for the first stage (if defined) +# 2. Builds a Tempest CR for each stage (reusing tempest-tests.yml) +# 3. Applies the first CR, pauses, then applies the rest +# 4. Waits for all pods to reach Succeeded or Failed +# 5. Collects logs from all pods +# 6. Records per-stage pass/fail results +# 7. Runs post_test_stage_hooks for the last stage (if defined) + +- name: "Run parallel group: {{ _parallel_stages | map(attribute='name') | join(', ') }}" + ansible.builtin.debug: + msg: >- + Running {{ _parallel_stages | length }} stages in parallel: + {{ _parallel_stages | map(attribute='name') | list | join(', ') }} + +- name: "Call pre stage hooks for parallel group" + when: + - not cifmw_test_operator_dry_run | bool + - _parallel_stages[0].pre_test_stage_hooks is defined + vars: + step: pre_test_hooks + hooks: "{{ _parallel_stages[0].pre_test_stage_hooks }}" + ansible.builtin.import_role: + name: run_hook + +- name: Initialize parallel CR list + ansible.builtin.set_fact: + _parallel_cr_list: [] + +- name: Build Tempest CRs for parallel group + ansible.builtin.include_tasks: build-parallel-cr.yml + loop: "{{ _parallel_stages }}" + loop_control: + loop_var: _pstage + extended: true + +- name: Log parallel CR names + ansible.builtin.debug: + msg: >- + Built {{ _parallel_cr_list | length }} CRs: + {{ _parallel_cr_list | map(attribute='metadata.name') | list | join(', ') }} + +- name: Parallel apply and wait block + when: not cifmw_test_operator_dry_run | bool + block: + - name: Make sure test-operator CR directory exists + ansible.builtin.file: + path: "{{ cifmw_test_operator_crs_path }}" + state: directory + mode: '0755' + + - name: Write all parallel CRs to disk + ansible.builtin.copy: + content: "{{ item | to_nice_yaml }}" + dest: "{{ cifmw_test_operator_crs_path }}/{{ item.metadata.name }}.yaml" + mode: '0644' + loop: "{{ _parallel_cr_list }}" + loop_control: + label: "{{ item.metadata.name }}" + + - name: "Apply first Tempest CR (creates shared images/flavors)" + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + definition: "{{ _parallel_cr_list[0] }}" + + - name: >- + Wait {{ cifmw_test_operator_parallel_resource_wait }}s + for first pod to create shared resources + ansible.builtin.pause: + seconds: "{{ cifmw_test_operator_parallel_resource_wait | int }}" + + - name: Apply remaining Tempest CRs + when: _parallel_cr_list | length > 1 + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + definition: "{{ item }}" + loop: "{{ _parallel_cr_list[1:] }}" + loop_control: + label: "{{ item.metadata.name }}" + + - name: "Wait for all {{ _parallel_cr_list | length }} Tempest pods to complete" + vars: + _parallel_timeout: >- + {{ cifmw_test_operator_parallel_timeout | default(cifmw_test_operator_timeout) }} + _parallel_prefix: >- + {{ _parallel_cr_list[0].metadata.name | regex_replace('-[^-]+$', '') }} + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + namespace: "{{ _parallel_cr_list[0].metadata.namespace }}" + kind: Pod + label_selectors: + - "service=tempest" + register: _parallel_pods + retries: "{{ ((_parallel_timeout | int) / 30) | int }}" + delay: 30 + until: >- + _parallel_pods.resources is defined and + (_parallel_pods.resources | + selectattr('metadata.labels.instanceName', 'defined') | + selectattr('metadata.labels.instanceName', 'in', + _parallel_cr_list | map(attribute='metadata.name') | list) | + selectattr('status.phase', 'defined') | + selectattr('status.phase', 'in', ['Succeeded', 'Failed']) | + list | length) >= (_parallel_cr_list | length) + ignore_errors: true + + - name: Check whether timed out + ansible.builtin.set_fact: + _parallel_timed_out: >- + {{ + _parallel_pods.attempts | default(0) >= + ((cifmw_test_operator_parallel_timeout | + default(cifmw_test_operator_timeout)) | int / 30) | int + }} + + - name: Collect logs for each parallel stage + when: not _parallel_timed_out + ansible.builtin.include_tasks: collect-parallel-logs.yml + loop: "{{ _parallel_cr_list }}" + loop_control: + loop_var: _pcr + label: "{{ _pcr.metadata.name }}" + + - name: Get results from all parallel test pods + vars: + _completed_pods: >- + {{ (_parallel_pods.resources | default([])) | + selectattr('metadata.labels.instanceName', 'defined') | + selectattr('metadata.labels.instanceName', 'in', + _parallel_cr_list | map(attribute='metadata.name') | list) | + list }} + _succeeded_pods: >- + {{ _completed_pods | + selectattr('status.phase', 'equalto', 'Succeeded') | list }} + _failed_pods: >- + {{ _completed_pods | + selectattr('status.phase', 'equalto', 'Failed') | list }} + block: + - name: Print parallel results summary + ansible.builtin.debug: + msg: | + === Parallel Tempest Results === + Total pods: {{ _completed_pods | length }} + Succeeded: {{ _succeeded_pods | length }} + Failed: {{ _failed_pods | length }} + Failed names: {{ _failed_pods | map(attribute='metadata.name') | list | join(', ') }} + + - name: Save per-stage results + ansible.builtin.set_fact: + test_operator_results: >- + {{ + test_operator_results | default({}) | + combine({ + 'tempest-' + item.metadata.labels.instanceName: + (item.status.phase == 'Succeeded') + }) + }} + loop: "{{ _completed_pods }}" + loop_control: + label: "{{ item.metadata.labels.instanceName }}" + + - name: Delete test resources + when: cifmw_test_operator_cleanup | bool + ansible.builtin.include_tasks: cleanup-parallel.yml + +- name: "Call post stage hooks for parallel group" + when: + - not cifmw_test_operator_dry_run | bool + - _parallel_stages[-1].post_test_stage_hooks is defined + vars: + step: post_test_hooks + hooks: "{{ _parallel_stages[-1].post_test_stage_hooks }}" + ansible.builtin.import_role: + name: run_hook diff --git a/roles/test_operator/tasks/stages.yml b/roles/test_operator/tasks/stages.yml index 675018677..8eba568ef 100644 --- a/roles/test_operator/tasks/stages.yml +++ b/roles/test_operator/tasks/stages.yml @@ -13,32 +13,8 @@ ansible.builtin.import_role: name: run_hook -- name: Set empty stage vars dict - ansible.builtin.set_fact: - stage_vars_dict: {} - -- name: Include stage var file - ansible.builtin.include_vars: - file: "{{ _stage_vars.test_vars_file | default('/dev/null') }}" - name: _stage_var_file - -- name: Merge file vars and test vars - vars: - file_vars: "{{ _stage_var_file }}" - test_vars: "{{ _stage_vars.test_vars | default({}) }}" - ansible.builtin.set_fact: - _stage_test_vars: "{{ file_vars | combine(test_vars) }}" - -- name: Overwrite global_vars with stage_vars - ansible.builtin.set_fact: - stage_vars_dict: "{{ stage_vars_dict | combine({item.key: _stage_test_vars[item.key] | default(lookup('vars', item.key, default=omit)) }) }}" - with_dict: "{{ vars | combine(_stage_test_vars) | dict2items | selectattr('key', 'match', '^cifmw_test_operator_') | items2dict }}" - -- name: Override specific type config - vars: - _stage_config: 'cifmw_test_operator_{{ _stage_vars.type }}_config' - ansible.builtin.set_fact: - stage_vars_dict: "{{ stage_vars_dict | combine({_stage_config: _stage_test_vars[_stage_config] | default(lookup('vars', _stage_config, default=omit)) }) }}" +- name: Load stage variables + ansible.builtin.include_tasks: load-stage-vars.yml - name: "Call runner {{ _stage_vars.type }}" ansible.builtin.include_tasks: "runners/{{ _stage_vars.type }}_runner.yml"