From d957578d7ac4390db014144f1c73f3b8328b1d37 Mon Sep 17 00:00:00 2001 From: Jiri Jaburek Date: Fri, 13 Feb 2026 12:45:29 +0100 Subject: [PATCH 1/3] remove Contest-based Fedora cTest execution This was likely a leftover from Beakerlib-era Fedora "downstream" testing - when we stopped doing it, we moved the only remaining valid test here. However since we run cTests via Github Actions in upstream, this extra test is likely unnecessary and complicates our Packit testing setup. Signed-off-by: Jiri Jaburek --- .packit.yaml | 5 ----- tests/tmt/plans/contest.fmf | 5 ----- 2 files changed, 10 deletions(-) diff --git a/.packit.yaml b/.packit.yaml index 8d456d18d276..d4c670b20ed6 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -36,11 +36,6 @@ jobs: # when modifying this, modify also tests/tmt-plans/ -- <<: *test-static-checks - identifier: /rpmbuild-ctest-fedora - tmt_plan: /plans/contest/rpmbuild-ctest-fedora$ - targets: - fedora-all: {} - <<: *test-static-checks identifier: /hardening/host-os/ansible/anssi_bp28_high tmt_plan: /plans/contest/hardening/host-os/ansible/anssi_bp28_high$ diff --git a/tests/tmt/plans/contest.fmf b/tests/tmt/plans/contest.fmf index 39056e491125..aceb2a6e306f 100644 --- a/tests/tmt/plans/contest.fmf +++ b/tests/tmt/plans/contest.fmf @@ -123,8 +123,3 @@ report: # The value of this test is debatable and therefore it should not delay upstream gating. # Our SCAP datastream is often noncompliant from the start, for example by containing SCE checks. - /static-checks/nist-validation - - -# Fedora specific plan -/rpmbuild-ctest-fedora: - discover+: {test: /static-checks/rpmbuild-ctest} From 801e12e0dd58de7bbc2b59875473d0e63884b30f Mon Sep 17 00:00:00 2001 From: Jiri Jaburek Date: Fri, 13 Feb 2026 12:50:38 +0100 Subject: [PATCH 2/3] unify the fedora-cis test under one directory Keeping plans/tests separate is not necessary, and the use case is isolated enough that it makes sense to keep all pieces of it together. Signed-off-by: Jiri Jaburek --- .packit.yaml | 2 +- tests/tmt/fedora-cis/main.fmf | 22 ++++++++++++++++++++++ tests/tmt/{tests => }/fedora-cis/test.sh | 0 tests/tmt/plans/fedora-cis.fmf | 7 ------- tests/tmt/tests/fedora-cis/main.fmf | 11 ----------- 5 files changed, 23 insertions(+), 19 deletions(-) create mode 100644 tests/tmt/fedora-cis/main.fmf rename tests/tmt/{tests => }/fedora-cis/test.sh (100%) delete mode 100644 tests/tmt/plans/fedora-cis.fmf delete mode 100644 tests/tmt/tests/fedora-cis/main.fmf diff --git a/.packit.yaml b/.packit.yaml index d4c670b20ed6..e40a632e546d 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -150,6 +150,6 @@ jobs: - <<: *test-static-checks identifier: fedora-cis - tmt_plan: /plans/fedora-cis$ + tmt_plan: /fedora-cis/plan$ targets: fedora-all: {} diff --git a/tests/tmt/fedora-cis/main.fmf b/tests/tmt/fedora-cis/main.fmf new file mode 100644 index 000000000000..2fe6663887e8 --- /dev/null +++ b/tests/tmt/fedora-cis/main.fmf @@ -0,0 +1,22 @@ +/plan: + discover: + how: fmf + test: /fedora-cis/test + execute: + how: tmt + report: + how: html + +/test: + summary: Runs oscap remediation using the CIS profile + description: |- + This ensures that the CIS profile is in a fairly good condition on + Fedora, to be used for ComplyTime demos, and as a baseline profile + for the community to target if they choose to adapt the ComplyTime + tooling. + test: ./test.sh + duration: 1h + require: + - openscap-scanner + - openscap-report + - scap-security-guide diff --git a/tests/tmt/tests/fedora-cis/test.sh b/tests/tmt/fedora-cis/test.sh similarity index 100% rename from tests/tmt/tests/fedora-cis/test.sh rename to tests/tmt/fedora-cis/test.sh diff --git a/tests/tmt/plans/fedora-cis.fmf b/tests/tmt/plans/fedora-cis.fmf deleted file mode 100644 index 7045aba0a895..000000000000 --- a/tests/tmt/plans/fedora-cis.fmf +++ /dev/null @@ -1,7 +0,0 @@ -discover: - how: fmf - test: /tests/fedora-cis -execute: - how: tmt -report: - how: html diff --git a/tests/tmt/tests/fedora-cis/main.fmf b/tests/tmt/tests/fedora-cis/main.fmf deleted file mode 100644 index f4f82cb0cd8e..000000000000 --- a/tests/tmt/tests/fedora-cis/main.fmf +++ /dev/null @@ -1,11 +0,0 @@ -summary: Runs oscap remediation using the CIS profile -description: |- - This ensures that the CIS profile is in a fairly good condition on Fedora, - to be used for ComplyTime demos, and as a baseline profile for the community - to target if they choose to adapt the ComplyTime tooling. -test: ./test.sh -duration: 1h -require: - - openscap-scanner - - openscap-report - - scap-security-guide From 67d48fcc7b0da159dc3b6481d43d95bb342f9053 Mon Sep 17 00:00:00 2001 From: Jiri Jaburek Date: Fri, 13 Feb 2026 12:54:55 +0100 Subject: [PATCH 3/3] combine Packit jobs running Contest The new layout has much fewer jobs: - centos-stream-8-x86_64:contest-oscap - centos-stream-8-x86_64:contest-ansible - centos-stream-9-x86_64:contest-oscap - centos-stream-9-x86_64:contest-ansible - centos-stream-10-x86_64:contest-oscap - centos-stream-10-x86_64:contest-ansible while keeping at least some separation for re-running. Within each job, all tests still execute in parallel, as parallel tmt plans, so there shouldn't be any extra performance hit or added delay. The coalescing will however vastly reduce the amount of Testing Farm "requests", hopefully reducing load on TF a lot, in addition to reducing load on Github runners. --- I opted for defining the plans on the Contest side instead of in tests/tmt/ because support for plan importing (as we did before) is limited and ie. doesn't allow filtering by tags. Having the plans in Contest allows us to automatically filter out profiles which are subsets of others, tests that always fail by design, etc., etc. We also don't need to worry about which tests/profiles are on which CentOS Stream, since Contest has "adjust" rules for that already, and an empty plan is automatically SKIPPED by Testing Farm. Signed-off-by: Jiri Jaburek --- .packit.yaml | 135 +++++------------------------------- tests/tmt/plans/contest.fmf | 125 --------------------------------- 2 files changed, 17 insertions(+), 243 deletions(-) delete mode 100644 tests/tmt/plans/contest.fmf diff --git a/.packit.yaml b/.packit.yaml index e40a632e546d..9b9987caac16 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -23,133 +23,32 @@ jobs: trigger: commit branch: "gh-readonly-queue/.*" -- &test-static-checks +- &contest-oscap job: tests trigger: pull_request - fmf_path: tests/tmt - identifier: /static-checks - tmt_plan: /plans/contest/static-checks$ + fmf_url: https://github.com/RHSecurityCompliance/contest.git + fmf_ref: main + tmt_plan: /plans/upstream-parallel/oscap + identifier: contest-oscap targets: centos-stream-8: {} centos-stream-9: {} centos-stream-10: {} + tf_extra_params: + settings: + pipeline: + parallel-limit: 32 -# when modifying this, modify also tests/tmt-plans/ +- <<: *contest-oscap + tmt_plan: /plans/upstream-parallel/ansible + identifier: contest-ansible -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/anssi_bp28_high - tmt_plan: /plans/contest/hardening/host-os/ansible/anssi_bp28_high$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/bsi - tmt_plan: /plans/contest/hardening/host-os/ansible/bsi$ - targets: - centos-stream-9: {} - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ccn_advanced - tmt_plan: /plans/contest/hardening/host-os/ansible/ccn_advanced$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis - tmt_plan: /plans/contest/hardening/host-os/ansible/cis$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_server_l1 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_server_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_workstation_l1 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_workstation_l2 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l2$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cui - tmt_plan: /plans/contest/hardening/host-os/ansible/cui$ - targets: - centos-stream-8: {} - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/e8 - tmt_plan: /plans/contest/hardening/host-os/ansible/e8$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/hipaa - tmt_plan: /plans/contest/hardening/host-os/ansible/hipaa$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ism_o - tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ism_o_top_secret - tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o_top_secret$ - targets: - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ospp - tmt_plan: /plans/contest/hardening/host-os/ansible/ospp$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/pci-dss - tmt_plan: /plans/contest/hardening/host-os/ansible/pci-dss$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/stig - tmt_plan: /plans/contest/hardening/host-os/ansible/stig$ - -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/anssi_bp28_high - tmt_plan: /plans/contest/hardening/host-os/oscap/anssi_bp28_high$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/bsi - tmt_plan: /plans/contest/hardening/host-os/oscap/bsi$ - targets: - centos-stream-9: {} - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ccn_advanced - tmt_plan: /plans/contest/hardening/host-os/oscap/ccn_advanced$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis - tmt_plan: /plans/contest/hardening/host-os/oscap/cis$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_server_l1 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_server_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_workstation_l1 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_workstation_l2 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l2$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cui - tmt_plan: /plans/contest/hardening/host-os/oscap/cui$ - targets: - centos-stream-8: {} - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/e8 - tmt_plan: /plans/contest/hardening/host-os/oscap/e8$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/hipaa - tmt_plan: /plans/contest/hardening/host-os/oscap/hipaa$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ism_o - tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ism_o_top_secret - tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o_top_secret$ - targets: - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ospp - tmt_plan: /plans/contest/hardening/host-os/oscap/ospp$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/pci-dss - tmt_plan: /plans/contest/hardening/host-os/oscap/pci-dss$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/stig - tmt_plan: /plans/contest/hardening/host-os/oscap/stig$ +# when modifying anything below, modify also tests/tmt/ -- <<: *test-static-checks - identifier: fedora-cis +- job: tests + trigger: pull_request + fmf_path: tests/tmt tmt_plan: /fedora-cis/plan$ + identifier: fedora-cis targets: fedora-all: {} diff --git a/tests/tmt/plans/contest.fmf b/tests/tmt/plans/contest.fmf deleted file mode 100644 index aceb2a6e306f..000000000000 --- a/tests/tmt/plans/contest.fmf +++ /dev/null @@ -1,125 +0,0 @@ -discover: - how: fmf - url: https://github.com/RHSecurityCompliance/contest.git -execute: - how: tmt -adjust: - - environment: - CONTEST_VERBOSE: 0 -report: - how: html - -# -# Hardening via ansible-playbook remediation -# - -/hardening/host-os/ansible/anssi_bp28_high: - discover+: {test: /hardening/host-os/ansible/anssi_bp28_high$} - -/hardening/host-os/ansible/bsi: - discover+: {test: /hardening/host-os/ansible/bsi$} - -/hardening/host-os/ansible/ccn_advanced: - discover+: {test: /hardening/host-os/ansible/ccn_advanced$} - -/hardening/host-os/ansible/cis: - discover+: {test: /hardening/host-os/ansible/cis$} - -/hardening/host-os/ansible/cis_server_l1: - discover+: {test: /hardening/host-os/ansible/cis_server_l1$} - -/hardening/host-os/ansible/cis_workstation_l1: - discover+: {test: /hardening/host-os/ansible/cis_workstation_l1$} - -/hardening/host-os/ansible/cis_workstation_l2: - discover+: {test: /hardening/host-os/ansible/cis_workstation_l2$} - -/hardening/host-os/ansible/cui: - discover+: {test: /hardening/host-os/ansible/cui$} - -/hardening/host-os/ansible/e8: - discover+: {test: /hardening/host-os/ansible/e8$} - -/hardening/host-os/ansible/hipaa: - discover+: {test: /hardening/host-os/ansible/hipaa$} - -/hardening/host-os/ansible/ism_o: - discover+: {test: /hardening/host-os/ansible/ism_o$} - -/hardening/host-os/ansible/ism_o_top_secret: - discover+: {test: /hardening/host-os/ansible/ism_o_top_secret$} - -/hardening/host-os/ansible/ospp: - discover+: {test: /hardening/host-os/ansible/ospp$} - -/hardening/host-os/ansible/pci-dss: - discover+: {test: /hardening/host-os/ansible/pci-dss$} - -/hardening/host-os/ansible/stig: - discover+: {test: /hardening/host-os/ansible/stig$} - -# -# Hardening via oscap xccdf eval --remediate -# - -/hardening/host-os/oscap/anssi_bp28_high: - discover+: {test: /hardening/host-os/oscap/anssi_bp28_high$} - -/hardening/host-os/oscap/bsi: - discover+: {test: /hardening/host-os/oscap/bsi$} - -/hardening/host-os/oscap/ccn_advanced: - discover+: {test: /hardening/host-os/oscap/ccn_advanced$} - -/hardening/host-os/oscap/cis: - discover+: {test: /hardening/host-os/oscap/cis$} - -/hardening/host-os/oscap/cis_server_l1: - discover+: {test: /hardening/host-os/oscap/cis_server_l1$} - -/hardening/host-os/oscap/cis_workstation_l1: - discover+: {test: /hardening/host-os/oscap/cis_workstation_l1$} - -/hardening/host-os/oscap/cis_workstation_l2: - discover+: {test: /hardening/host-os/oscap/cis_workstation_l2$} - -/hardening/host-os/oscap/cui: - discover+: {test: /hardening/host-os/oscap/cui$} - -/hardening/host-os/oscap/e8: - discover+: {test: /hardening/host-os/oscap/e8$} - -/hardening/host-os/oscap/hipaa: - discover+: {test: /hardening/host-os/oscap/hipaa$} - -/hardening/host-os/oscap/ism_o: - discover+: {test: /hardening/host-os/oscap/ism_o$} - -/hardening/host-os/oscap/ism_o_top_secret: - discover+: {test: /hardening/host-os/oscap/ism_o_top_secret$} - -/hardening/host-os/oscap/ospp: - discover+: {test: /hardening/host-os/oscap/ospp$} - -/hardening/host-os/oscap/pci-dss: - discover+: {test: /hardening/host-os/oscap/pci-dss$} - -/hardening/host-os/oscap/stig: - discover+: {test: /hardening/host-os/oscap/stig$} - -# -# Misc smoke/sanity tests -# - -/static-checks: - discover+: - test: /static-checks - exclude: - # exclude here due to the test failing frequently for short periods - # of time, as many websites have temporary availability issues - - /static-checks/html-links - # these always fail, meant for manual review - - /static-checks/diff - # The value of this test is debatable and therefore it should not delay upstream gating. - # Our SCAP datastream is often noncompliant from the start, for example by containing SCE checks. - - /static-checks/nist-validation