[DNM] feat(test_operator): add parallel_group support for concurrent tempest stages

roles/test_operator/defaults/main.yml

@@ -87,6 +87,23 @@ cifmw_test_operator_tempest_resources:
   requests: {}
   limits: {}
 
+# Section 1b: parallel execution parameters
+# Used when stages share a parallel_group value.
+# Seconds to wait after the first CR is applied, allowing shared
+# resources (images, flavors) to be created before remaining CRs start.
+cifmw_test_operator_parallel_resource_wait: 120
+# Maximum seconds to wait for all parallel pods to complete.
+cifmw_test_operator_parallel_timeout: 14400
+# Resource requests/limits applied to each pod in a parallel group.
+# Smaller than single-stage defaults so multiple pods fit on one node.
+cifmw_test_operator_parallel_resources:
+  requests:
+    cpu: "500m"
+    memory: "512Mi"
+  limits:
+    cpu: "2"
+    memory: "2Gi"
+
 # Enabling SRBAC by default, in jobs where this does not make sense should be turned off explicitly
 #
 # auth.tempest_roles is set to an empty value because otherwise

roles/test_operator/tasks/build-parallel-cr.yml

@@ -0,0 +1,99 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Build a single Tempest CR for a parallel stage.
+#
+# This reuses the standard tempest-tests.yml logic (include/exclude lists,
+# SSH key secret, controller IP injection, workflow overrides, etc.) to
+# construct the CR, then applies parallel-specific modifications.
+#
+# Expected input:
+#   _pstage:              stage dict (name, type, test_vars, etc.)
+#   ansible_loop.index0:  position in the parallel group (0-based)
+#
+# Side effect:
+#   Appends the built CR to _parallel_cr_list fact.
+
+- name: "Build parallel CR for stage: {{ _pstage.name }}"
+  ansible.builtin.debug:
+    msg: "Building CR {{ ansible_loop.index0 + 1 }}/{{ ansible_loop.length }}: {{ _pstage.name }}"
+
+- name: "Load stage variables for {{ _pstage.name }}"
+  vars:
+    _stage_vars: "{{ _pstage }}"
+  ansible.builtin.include_tasks: load-stage-vars.yml
+
+- name: "Set CR base from tempest config template for {{ _pstage.name }}"
+  vars:
+    _stage_vars: "{{ _pstage }}"
+  ansible.builtin.set_fact:
+    test_operator_cr: "{{ stage_vars_dict.cifmw_test_operator_tempest_config }}"
+
+- name: "Clear inherited workflow for parallel stage {{ _pstage.name }}"
+  ansible.builtin.set_fact:
+    stage_vars_dict: >-
+      {{
+        stage_vars_dict | combine({
+          'cifmw_test_operator_tempest_workflow': []
+        })
+      }}
+
+- name: "Apply tempest-specific configuration for {{ _pstage.name }}"
+  vars:
+    _stage_vars: "{{ _pstage }}"
+    run_test_fw: tempest
+    test_operator_instance_name: >-
+      {{ stage_vars_dict.cifmw_test_operator_tempest_name }}-{{ _pstage.name }}
+    test_operator_workflow: []
+  ansible.builtin.include_tasks: tempest-tests.yml
+
+- name: "Apply parallel-specific overrides for {{ _pstage.name }}"
+  ansible.builtin.set_fact:
+    test_operator_cr: >-
+      {{
+        test_operator_cr | combine({
+          'spec': {
+            'parallel': true,
+            'workflow': [],
+            'resources': cifmw_test_operator_parallel_resources |
+                         default(test_operator_cr.spec.resources | default({})),
+            'tempestconfRun':
+              test_operator_cr.spec.tempestconfRun | default({}) | combine({
+                'create': (ansible_loop.index0 == 0)
+              }),
+            'tempestRun':
+              test_operator_cr.spec.tempestRun | default({}) | combine({
+                'extraImages': (
+                  test_operator_cr.spec.tempestRun.extraImages | default([])
+                  if ansible_loop.index0 == 0
+                  else [])
+              }),
+            'rerunFailedTests':
+              _pstage.rerunFailedTests | default(
+                stage_vars_dict.cifmw_test_operator_tempest_rerun_failed_tests |
+                default(false)) | bool,
+            'rerunOverrideStatus':
+              _pstage.rerunOverrideStatus | default(
+                stage_vars_dict.cifmw_test_operator_tempest_rerun_override_status |
+                default(false)) | bool,
+          }
+        }, recursive=true)
+      }}
+
+- name: "Append CR to parallel list: {{ _pstage.name }}"
+  ansible.builtin.set_fact:
+    _parallel_cr_list: >-
+      {{ _parallel_cr_list | default([]) + [test_operator_cr] }}

roles/test_operator/tasks/cleanup-parallel.yml

@@ -0,0 +1,50 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Clean up resources created by parallel Tempest CRs.
+#
+# Expected input:
+#   _parallel_cr_list: list of Tempest CR definition dicts
+
+- name: Delete parallel Tempest CRs
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    state: absent
+    definition: "{{ item }}"
+    wait: true
+    wait_timeout: 600
+  loop: "{{ _parallel_cr_list }}"
+  loop_control:
+    label: "{{ item.metadata.name }}"
+
+- name: Delete parallel log pods
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    state: absent
+    api_version: v1
+    kind: Pod
+    name: "test-operator-logs-pod-tempest-{{ item.metadata.name }}"
+    namespace: "{{ item.metadata.namespace }}"
+    wait: true
+    wait_timeout: 600
+  when: cifmw_test_operator_delete_logs_pod | bool or cifmw_test_operator_cleanup | bool
+  loop: "{{ _parallel_cr_list }}"
+  loop_control:
+    label: "{{ item.metadata.name }}"

roles/test_operator/tasks/collect-parallel-logs.yml

@@ -0,0 +1,158 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Collect logs from a single parallel Tempest CR's PVCs.
+#
+# Reuses the same PVC-based log collection approach as collect-logs.yaml
+# but scoped to a specific parallel CR instance.
+#
+# Expected input:
+#   _pcr: the Tempest CR definition dict (with .metadata.name, .metadata.namespace)
+
+- name: "Reset volumes for {{ _pcr.metadata.name }}"
+  ansible.builtin.set_fact:
+    _test_operator_volumes: []
+    _test_operator_volume_mounts: []
+
+- name: "Get PVCs for {{ _pcr.metadata.name }}"
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    namespace: "{{ _pcr.metadata.namespace }}"
+    kind: PersistentVolumeClaim
+    label_selectors:
+      - "instanceName={{ _pcr.metadata.name }}"
+  register: _pcr_pvcs
+
+- name: "Set up volume mounts for {{ _pcr.metadata.name }}"
+  ansible.builtin.set_fact:
+    _test_operator_volume_mounts: >-
+      {{
+        (_test_operator_volume_mounts | default([])) + [{
+          'name': 'logs-volume-' ~ index,
+          'mountPath': '/mnt/logs-' + _pcr.metadata.name + '-step-' ~ index
+        }]
+      }}
+    _test_operator_volumes: >-
+      {{
+        (_test_operator_volumes | default([])) + [{
+          'name': 'logs-volume-' ~ index,
+          'persistentVolumeClaim': {
+            'claimName': pvc.metadata.name
+          }
+        }]
+      }}
+  loop: "{{ _pcr_pvcs.resources }}"
+  loop_control:
+    loop_var: pvc
+    index_var: index
+
+- name: "Create log pod definition for {{ _pcr.metadata.name }}"
+  when: _pcr_pvcs.resources | length > 0
+  block:
+    - name: Set log pod fact
+      vars:
+        run_test_fw: tempest
+        test_operator_instance_name: "{{ _pcr.metadata.name }}"
+      ansible.builtin.set_fact:
+        _test_operator_log_pod: "{{ cifmw_test_operator_log_pod_definition }}"
+
+    - name: "Write log pod definition for {{ _pcr.metadata.name }}"
+      ansible.builtin.copy:
+        content: "{{ _test_operator_log_pod | to_nice_yaml }}"
+        dest: "{{ cifmw_test_operator_crs_path }}/{{ _pcr.metadata.name }}-log-pod.yaml"
+        mode: '0644'
+
+    - name: "Start log pod for {{ _pcr.metadata.name }}"
+      kubernetes.core.k8s:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        api_key: "{{ cifmw_openshift_token | default(omit) }}"
+        context: "{{ cifmw_openshift_context | default(omit) }}"
+        state: present
+        wait: true
+        src: "{{ cifmw_test_operator_crs_path }}/{{ _pcr.metadata.name }}-log-pod.yaml"
+
+    - name: "Ensure log pod is Running for {{ _pcr.metadata.name }}"
+      kubernetes.core.k8s_info:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        api_key: "{{ cifmw_openshift_token | default(omit) }}"
+        context: "{{ cifmw_openshift_context | default(omit) }}"
+        namespace: "{{ _pcr.metadata.namespace }}"
+        kind: Pod
+        name: "test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}"
+        wait: true
+      register: _pcr_logs_pod
+      until: _pcr_logs_pod.resources[0].status.phase == "Running"
+      delay: 10
+      retries: 20
+
+    - name: "Copy logs from {{ _pcr.metadata.name }}"
+      environment:
+        KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+        PATH: "{{ cifmw_path }}"
+      vars:
+        _pod_path: "mnt/logs-{{ _pcr.metadata.name }}-step-{{ index }}"
+      ansible.builtin.shell: >-
+        oc cp -n {{ _pcr.metadata.namespace }}
+        test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}:{{ _pod_path }}
+        {{ cifmw_test_operator_artifacts_basedir }}
+      loop: "{{ _pcr_pvcs.resources }}"
+      loop_control:
+        index_var: index
+
+    - name: "Find subunit files for {{ _pcr.metadata.name }}"
+      failed_when: false
+      ansible.builtin.find:
+        paths: "{{ cifmw_test_operator_artifacts_basedir }}"
+        patterns: "*.subunit"
+        recurse: true
+      register: _pcr_subunit_files
+
+    - name: Install subunit and stestr packages
+      become: true
+      when: _pcr_subunit_files.files | default([]) | length > 0
+      failed_when: false
+      ansible.builtin.dnf:
+        name:
+          - python3-subunit
+          - python3-stestr
+        state: present
+
+    - name: "Generate HTML reports for {{ _pcr.metadata.name }}"
+      when: _pcr_subunit_files.files | default([]) | length > 0
+      failed_when: false
+      ansible.builtin.command:
+        cmd: >-
+          python3 {{ role_path }}/files/subunit-to-html.py
+          {{ item.path }}
+          {{ item.path | regex_replace('\.subunit$', '-viz.html') }}
+      loop: "{{ _pcr_subunit_files.files }}"
+      loop_control:
+        label: "{{ item.path | basename }}"
+
+    - name: "Delete log pod for {{ _pcr.metadata.name }}"
+      kubernetes.core.k8s:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        api_key: "{{ cifmw_openshift_token | default(omit) }}"
+        context: "{{ cifmw_openshift_context | default(omit) }}"
+        state: absent
+        api_version: v1
+        kind: Pod
+        name: "test-operator-logs-pod-tempest-{{ _pcr.metadata.name }}"
+        namespace: "{{ _pcr.metadata.namespace }}"
+        wait: true
+        wait_timeout: 120

roles/test_operator/tasks/dispatch-stage.yml

@@ -0,0 +1,62 @@
+---
+# Copyright Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Dispatch a single stage to either sequential or parallel execution.
+#
+# Stages with parallel_group are collected and executed together the first
+# time the group is encountered. Subsequent stages in the same group are
+# skipped (already processed).
+#
+# Stages without parallel_group are executed sequentially as before.
+#
+# Expected input:
+#   _stage_vars: the current stage dict from the main loop
+
+- name: "Process parallel group: {{ _stage_vars.parallel_group }}"
+  when:
+    - _stage_vars.parallel_group is defined
+    - _stage_vars.parallel_group not in (_processed_parallel_groups | default([]))
+  block:
+    - name: "Collect all stages in parallel group: {{ _stage_vars.parallel_group }}"
+      ansible.builtin.set_fact:
+        _parallel_stages: >-
+          {{
+            cifmw_test_operator_stages |
+            selectattr('parallel_group', 'defined') |
+            selectattr('parallel_group', 'equalto', _stage_vars.parallel_group) |
+            list
+          }}
+
+    - name: Execute parallel stages
+      ansible.builtin.include_tasks: parallel_stages.yml
+
+    - name: Mark parallel group as processed
+      ansible.builtin.set_fact:
+        _processed_parallel_groups: >-
+          {{ (_processed_parallel_groups | default([])) + [_stage_vars.parallel_group] }}
+
+- name: "Skip already-processed parallel stage: {{ _stage_vars.name }}"
+  when:
+    - _stage_vars.parallel_group is defined
+    - _stage_vars.parallel_group in (_processed_parallel_groups | default([]))
+  ansible.builtin.debug:
+    msg: >-
+      Skipping {{ _stage_vars.name }} - parallel group
+      {{ _stage_vars.parallel_group }} already processed.
+
+- name: "Process sequential stage: {{ _stage_vars.name }}"
+  when: _stage_vars.parallel_group is not defined
+  ansible.builtin.include_tasks: stages.yml