WIP: Use group_vars in ci/playbooks

ci/playbooks/architecture/run.yml

@@ -3,6 +3,18 @@
   hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Filter out host if needed
       when:
         - cifmw_zuul_target_host is defined
@@ -12,7 +24,7 @@
 
     - name: Run playbook
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook -i localhost, -c local
           ci/playbooks/architecture/validate-architecture.yml

ci/playbooks/bootstrap-networking-mapper.yml

@@ -3,6 +3,18 @@
   hosts: controller
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Create a networking_mapper playbook
       ansible.builtin.copy:
         dest: "{{ ansible_user_dir }}/networking_mapper.yml"
@@ -17,7 +29,7 @@
       environment:
         ANSIBLE_HOST_KEY_CHECKING: false
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ~/test-python/bin/ansible-playbook {{ ansible_user_dir }}/networking_mapper.yml
           -i {{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml

ci/playbooks/build_push_container_runner.yml

@@ -2,9 +2,21 @@
 - name: Run build_push_container role
   hosts: "{{ cifmw_zuul_target_host | default('controller') }}"
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Run build_push_container role
       vars:
-        ci_framework_src_dir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        ci_framework_src_dir: "{{ cifmw_project_dir_absolute }}"
       ansible.builtin.command:
         chdir: "{{ ci_framework_src_dir }}"
         cmd: >-

ci/playbooks/content_provider/pre.yml

@@ -1,4 +1,12 @@
 ---
+- name: Get all repos on all hosts
+  hosts: all
+  gather_facts: true
+  tasks:
+    - name: Clone repos in the job workspace
+      ansible.builtin.include_role:
+        name: prepare-workspace
+
 - name: "Run ci/playbooks/content_provider/pre.yml"
   hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   tasks:
@@ -9,10 +17,6 @@
         - inventory_hostname != cifmw_zuul_target_host
       ansible.builtin.meta: end_host
 
-    - name: Clone repos in the job workspace
-      ansible.builtin.include_role:
-        name: prepare-workspace
-
     - name: Install ansible-core
       become: true
       ansible.builtin.package:

ci/playbooks/content_provider/run.yml

@@ -3,6 +3,18 @@
   hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Filter out host if needed
       when:
         - cifmw_zuul_target_host is defined
@@ -12,9 +24,9 @@
 
     - name: Deploy content provider
       environment:
-        ANSIBLE_CONFIG: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/ansible.cfg"
+        ANSIBLE_CONFIG: "{{ cifmw_project_dir_absolute }}/ansible.cfg"
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook -i localhost,
           -c local ci/playbooks/content_provider/content_provider.yml

ci/playbooks/e2e-collect-logs.yml

@@ -3,6 +3,18 @@
   hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Filter out host if needed
       when:
         - cifmw_zuul_target_host is defined
@@ -13,7 +25,7 @@
     - name: Ensure we have the ci-framework on host
       register: cifmw_status
       ansible.builtin.stat:
-        path: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        path: "{{ cifmw_project_dir_absolute }}"
 
     - name: End host if no basedir
       when:
@@ -35,7 +47,7 @@
         tasks_from: run_logs.yml
       tags:
         - logs
-      environment:
+     environment:
         ANSIBLE_LOG_PATH: "{{ ansible_user_dir }}/ci-framework-data/logs/e2e-collect-logs-must-gather.log"
 
 - name: "Run ci/playbooks/e2e-collect-logs.yml on CRC host"

ci/playbooks/e2e-prepare.yml

@@ -1,12 +1,28 @@
 ---
-- name: "Run ci/playbooks/e2e-prepare.yml"
-  hosts: controller, primary
+- name: Get all repos on all hosts
+  hosts: all
   gather_facts: true
   tasks:
     - name: Clone repos in the job workspace
       ansible.builtin.include_role:
         name: prepare-workspace
 
+- name: "Run ci/playbooks/e2e-prepare.yml"
+  hosts: controller, primary
+  gather_facts: true
+  tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Create zuul-output directory
       ansible.builtin.file:
         path: "{{ ansible_user_dir }}/zuul-output/logs"
@@ -26,9 +42,13 @@
       ansible.builtin.command:
         cmd: pip --version
 
+    - name: DNM Print dir
+      ansible.builtin.debug:
+        msg: "proj dir is {{ cifmw_project_dir_absolute }}"
+
     - name: Install requirements
       community.general.make:
-        chdir: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         target: setup_molecule
 
     - name: Add cronjob to trigger job stats
@@ -37,7 +57,7 @@
         minute: "*"
         job: >-
           /usr/bin/date >> /tmp/crio-stats.log;
-          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/scripts/get-stats.sh >>
+          {{ cifmw_project_dir_absolute }}/scripts/get-stats.sh >>
           /tmp/crio-stats.log
       when: cifmw_openshift_crio_stats | default(false)
 

ci/playbooks/e2e-run.yml

@@ -2,12 +2,22 @@
 - name: "Run ci/playbooks/e2e-run.yml"
   hosts: "{{ cifmw_zuul_target_host | default('controller') }}"
   gather_facts: true
-  vars:
-    ci_framework_src_dir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Tagged packages
       ansible.builtin.command:
-        chdir: "{{ ci_framework_src_dir }}"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
           -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
@@ -28,7 +38,7 @@
 
     - name: Tagged bootstrap - no packages
       ansible.builtin.command:
-        chdir: "{{ ci_framework_src_dir }}"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
           -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
@@ -50,7 +60,7 @@
 
     - name: Not tagged bootstrap nor packages
       ansible.builtin.command:
-        chdir: "{{ ci_framework_src_dir }}"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
           -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"

ci/playbooks/edpm/run.yml

@@ -3,6 +3,18 @@
   hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Filter out host if needed
       when:
         - cifmw_zuul_target_host is defined
@@ -24,7 +36,7 @@
 
     - name: Run Podified EDPM deployment
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
           -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
@@ -42,7 +54,7 @@
 
     - name: Run Podified EDPM post deployment
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook post-deployment.yml
           -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"

ci/playbooks/edpm/update.yml

@@ -3,6 +3,18 @@
   hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Filter out host if needed
       when:
         - cifmw_zuul_target_host is defined
@@ -17,7 +29,7 @@
 
     - name: Run Podified EDPM update
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook update-edpm.yml
           -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"

ci/playbooks/edpm_baremetal_deployment/run.yml

@@ -3,6 +3,18 @@
   hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Filter out host if needed
       when:
         - cifmw_zuul_target_host is defined
@@ -55,7 +67,7 @@
 
     - name: Perform Podified and EDPM deployment on compute nodes with virtual baremetal
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
           -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"

ci/playbooks/edpm_build_images/edpm_build_images_content_provider_run.yaml

@@ -1,8 +1,20 @@
 ---
 - name: "Run ci/playbooks/edpm_build_images/edpm_build_images_content_provider_run.yaml"
-  hosts: "{{ cifmw_zuul_target_host | default('all') }}"
+  hosts: "{{ cifmw_zuul_target_host | default('all') }}"
   gather_facts: true
   tasks:
+    # FIXME(amsinha/danpawlik): Current version of zuul does not support include-vars [1] from file,
+    # thus using this hack to read files. Will replace all these hacky var file read with include-vars
+    # once Zuul is upgraded.
+    # [1] https://zuul-ci.org/docs/zuul/latest/config/job.html#attr-job.include-vars
+    - name: Read group_vars
+      vars:
+        provided_dir: >
+          {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/group_vars
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: var_dir.yml
+
     - name: Filter out host if needed
       when:
         - cifmw_zuul_target_host is defined
@@ -12,9 +24,9 @@
 
     - name: Build edpm images
       environment:
-        ANSIBLE_CONFIG: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/ansible.cfg"
+        ANSIBLE_CONFIG: "{{ cifmw_project_dir_absolute }}/ansible.cfg"
       ansible.builtin.command:
-        chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
+        chdir: "{{ cifmw_project_dir_absolute }}"
         cmd: >-
           ansible-playbook -i localhost, -c local
           ci/playbooks/edpm_build_images/edpm_build_images_content_provider.yaml