diff --git a/.github/workflows/atex-build.yaml b/.github/workflows/atex-build.yaml index bfc295674bc4..84aa1400ff96 100644 --- a/.github/workflows/atex-build.yaml +++ b/.github/workflows/atex-build.yaml @@ -63,12 +63,24 @@ jobs: # Clean up temporary metadata rm -rf jinja2_cache + - name: Save file permissions before artifact upload + run: | + # GitHub Actions artifact upload/download strips execute permissions + # Save all file permissions so they can be restored after download + echo "=== Saving file permissions ===" + find . -type f -printf '%m %p\n' > file-permissions.txt + echo "Saved permissions for $(wc -l < file-permissions.txt) files" + # Show sample of executable files being saved + echo "=== Sample executable files ===" + grep -E '^[0-7]*[1357][0-7]* ' file-permissions.txt | head -10 || true + - name: Upload build artifacts uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: content-centos-stream${{ matrix.centos_stream_major }} path: . retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} + include-hidden-files: true # make sure all .dot files are included e.g. .cmakelintrc save_pr_info: name: Save PR information for workflow_run diff --git a/.github/workflows/atex-test.yaml b/.github/workflows/atex-test.yaml index 86bb59cbaa46..b0f229fa2874 100644 --- a/.github/workflows/atex-test.yaml +++ b/.github/workflows/atex-test.yaml @@ -12,6 +12,9 @@ env: CONTEST_REPO: RHSecurityCompliance/contest ARTIFACT_RETENTION_DAYS: 1 TEST_TIMEOUT: 1440 # 24 hours + # CentOS Stream versions to test (space-separated for shell loops) + # NOTE: Keep in sync with matrix.centos_stream_major in the test job + CS_VERSIONS: "8 9 10" permissions: contents: read @@ -66,8 +69,14 @@ jobs: name: Test on CentOS Stream ${{ matrix.centos_stream_major }} runs-on: ubuntu-latest needs: check_build + outputs: + # Contest SHA from any matrix job (all use same ref, so same SHA) + contest_sha: ${{ steps.get_contest.outputs.contest_sha }} + contest_ref: ${{ steps.get_contest.outputs.contest_ref }} strategy: + fail-fast: false matrix: + # NOTE: Keep in sync with env.CS_VERSIONS at the top of this file centos_stream_major: [8, 9, 10] container: image: fedora:latest @@ -84,6 +93,36 @@ jobs: name: content-centos-stream${{ matrix.centos_stream_major }} path: content-centos-stream${{ matrix.centos_stream_major }}/ + - name: Restore file permissions lost during artifact download + run: | + # GitHub Actions artifact download strips execute permissions + # Restore permissions from the saved file created during build + CONTENT_DIR="content-centos-stream${{ matrix.centos_stream_major }}" + PERMS_FILE="${CONTENT_DIR}/file-permissions.txt" + + if [ -f "${PERMS_FILE}" ]; then + echo "=== Restoring file permissions from ${PERMS_FILE} ===" + cd "${CONTENT_DIR}" + while IFS=' ' read -r mode filepath; do + # Remove leading ./ from filepath if present + filepath="${filepath#./}" + if [ -f "${filepath}" ]; then + chmod "${mode}" "${filepath}" + fi + done < file-permissions.txt + echo "Restored permissions for $(wc -l < file-permissions.txt) files" + # Show sample of restored executable files + echo "=== Sample executable files after restore ===" + find . -type f -executable -name "*.py" 2>/dev/null | head -5 || true + find . -type f -executable -name "*.sh" 2>/dev/null | head -5 || true + else + echo "WARNING: ${PERMS_FILE} not found, permissions may be incorrect" + exit 1 + fi + + - name: Install git for checkout + run: dnf -y install git + - name: Checkout Contest Test Suite uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: @@ -92,10 +131,19 @@ jobs: path: contest fetch-depth: 1 + - name: Get Contest SHA + id: get_contest + run: | + CONTEST_SHA=$(cd contest && git rev-parse HEAD) + CONTEST_REF="main" + echo "contest_sha=${CONTEST_SHA}" >> $GITHUB_OUTPUT + echo "contest_ref=${CONTEST_REF}" >> $GITHUB_OUTPUT + echo "Contest: ${CONTEST_SHA:0:12} (${CONTEST_REF})" + - name: Install test dependencies run: | - dnf -y install python3-pip git rsync - pip install fmf atex==0.11 + dnf -y install python3-pip rsync + pip install fmf atex==0.12 - name: Run tests on Testing Farm env: @@ -105,8 +153,7 @@ jobs: python3 tests/run_tests_testingfarm.py \ --contest-dir contest \ --content-dir content-centos-stream${CS_MAJOR} \ - --plan "/plans/daily" \ - --tests "/hardening/host-os/oscap/stig" \ + --plan "/plans/upstream" \ --compose "CentOS-Stream-${CS_MAJOR}" \ --arch x86_64 \ --os-major-version "${CS_MAJOR}" \ @@ -139,7 +186,7 @@ jobs: if: always() run: | dnf -y install python3-pip git rsync - pip install fmf atex==0.11 + pip install fmf atex==0.12 - name: Checkout ATEX results repository if: always() @@ -155,16 +202,16 @@ jobs: working-directory: atex-results-testing-farm run: fmf init - - name: Create TMT dummy plan for artifact transport + - name: Create TMT atex_results plan for artifact transport if: always() working-directory: atex-results-testing-farm run: | cat > main.fmf <<'EOF' - /dummy_plan: + /atex_results_plan: discover: how: shell tests: - - name: /dummy_test + - name: /atex_results_test test: mv * "$TMT_TEST_DATA/." execute: how: tmt @@ -217,7 +264,7 @@ jobs: mkdir -p atex-results-testing-farm/files_dir/ # Process and merge results for all CentOS Stream versions - for version in 8 9 10; do + for version in ${{ env.CS_VERSIONS }}; do results_file="test-results/cs${version}/results-centos-stream-${version}-x86_64.json.xz" files_dir="test-results/cs${version}/files-centos-stream-${version}-x86_64" @@ -238,6 +285,74 @@ jobs: run: | cp -rf atex-html/index.html atex-html/sqljs/ atex-results-testing-farm/ + - name: Generate header.html for results page + if: always() + env: + PR_NUMBER: ${{ needs.check_build.outputs.pr_number }} + PR_SHA: ${{ needs.check_build.outputs.pr_sha }} + CONTEST_SHA: ${{ needs.test.outputs.contest_sha }} + CONTEST_REF: ${{ needs.test.outputs.contest_ref }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + REPO_URL: ${{ github.server_url }}/${{ github.repository }} + ACTOR: ${{ github.actor }} + RUN_STARTED: ${{ github.event.workflow_run.created_at }} + CS_VERSIONS: ${{ env.CS_VERSIONS }} + run: | + cat > atex-results-testing-farm/header.html <<'HEADER_EOF' + + HEADER_EOF + + # Add dynamic content - header section + cat >> atex-results-testing-farm/header.html <ATEX Upstream Testing +

PR #${PR_NUMBER} + - Workflow #${{ github.run_id }} + started on + by ${ACTOR}

+ +
+ + + EOF + + # List each CentOS Stream version that was tested + for version in ${CS_VERSIONS}; do + echo " " >> atex-results-testing-farm/header.html + done + + # Add commit info table + cat >> atex-results-testing-farm/header.html < +
CentOS Stream
${version}
+ + + +
RepoCommit used
Content${PR_SHA:0:12}
Contest (${CONTEST_REF})${CONTEST_SHA:0:12}
+
+ EOF + + echo "=== Generated header.html ===" + cat atex-results-testing-farm/header.html + - name: Commit and tag results in ATEX repository if: always() working-directory: atex-results-testing-farm @@ -311,6 +426,7 @@ jobs: check_id: ${{ needs.check_build.outputs.check_id }} sha: ${{ needs.check_build.outputs.pr_sha }} status: completed - conclusion: ${{ job.status }} + # Use test job result to determine conclusion - needs.test.result will be 'failure' if any matrix job failed + conclusion: ${{ needs.test.result }} output: | {"summary":"ATEX tests completed. Job: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. View results: ${{ steps.testing_farm_request.outputs.HTML_LINK }}","title":"ATEX Testing Complete"} diff --git a/.packit.yaml b/.packit.yaml index 2e9ab02c8b3a..9692e98b6610 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -23,136 +23,16 @@ jobs: trigger: commit branch: "gh-readonly-queue/.*" -- &test-static-checks +# when modifying this, modify also tests/tmt-plans/ +- &fedora-tests job: tests trigger: pull_request fmf_path: tests/tmt - identifier: /static-checks - tmt_plan: /plans/contest/static-checks$ - targets: - centos-stream-8: {} - centos-stream-9: {} - centos-stream-10: {} - -# when modifying this, modify also tests/tmt-plans/ - -- <<: *test-static-checks identifier: /rpmbuild-ctest-fedora tmt_plan: /plans/contest/rpmbuild-ctest-fedora$ targets: fedora-all: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/anssi_bp28_high - tmt_plan: /plans/contest/hardening/host-os/ansible/anssi_bp28_high$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/bsi - tmt_plan: /plans/contest/hardening/host-os/ansible/bsi$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ccn_advanced - tmt_plan: /plans/contest/hardening/host-os/ansible/ccn_advanced$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis - tmt_plan: /plans/contest/hardening/host-os/ansible/cis$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_server_l1 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_server_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_workstation_l1 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cis_workstation_l2 - tmt_plan: /plans/contest/hardening/host-os/ansible/cis_workstation_l2$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/cui - tmt_plan: /plans/contest/hardening/host-os/ansible/cui$ - targets: - centos-stream-8: {} - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/e8 - tmt_plan: /plans/contest/hardening/host-os/ansible/e8$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/hipaa - tmt_plan: /plans/contest/hardening/host-os/ansible/hipaa$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ism_o - tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ism_o_top_secret - tmt_plan: /plans/contest/hardening/host-os/ansible/ism_o_top_secret$ - targets: - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/ospp - tmt_plan: /plans/contest/hardening/host-os/ansible/ospp$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/pci-dss - tmt_plan: /plans/contest/hardening/host-os/ansible/pci-dss$ -- <<: *test-static-checks - identifier: /hardening/host-os/ansible/stig - tmt_plan: /plans/contest/hardening/host-os/ansible/stig$ - -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/anssi_bp28_high - tmt_plan: /plans/contest/hardening/host-os/oscap/anssi_bp28_high$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/bsi - tmt_plan: /plans/contest/hardening/host-os/oscap/bsi$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ccn_advanced - tmt_plan: /plans/contest/hardening/host-os/oscap/ccn_advanced$ - targets: - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis - tmt_plan: /plans/contest/hardening/host-os/oscap/cis$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_server_l1 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_server_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_workstation_l1 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l1$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cis_workstation_l2 - tmt_plan: /plans/contest/hardening/host-os/oscap/cis_workstation_l2$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/cui - tmt_plan: /plans/contest/hardening/host-os/oscap/cui$ - targets: - centos-stream-8: {} - centos-stream-9: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/e8 - tmt_plan: /plans/contest/hardening/host-os/oscap/e8$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/hipaa - tmt_plan: /plans/contest/hardening/host-os/oscap/hipaa$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ism_o - tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ism_o_top_secret - tmt_plan: /plans/contest/hardening/host-os/oscap/ism_o_top_secret$ - targets: - centos-stream-10: {} -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/ospp - tmt_plan: /plans/contest/hardening/host-os/oscap/ospp$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/pci-dss - tmt_plan: /plans/contest/hardening/host-os/oscap/pci-dss$ -- <<: *test-static-checks - identifier: /hardening/host-os/oscap/stig - tmt_plan: /plans/contest/hardening/host-os/oscap/stig$ -- <<: *test-static-checks +- <<: *fedora-tests identifier: fedora-cis tmt_plan: /plans/fedora-cis$ - targets: - fedora-all: {} diff --git a/tests/run_tests_testingfarm.py b/tests/run_tests_testingfarm.py index 310b6fcc4dbd..5913993bfece 100644 --- a/tests/run_tests_testingfarm.py +++ b/tests/run_tests_testingfarm.py @@ -3,6 +3,10 @@ import sys import time import gzip +import json +import lzma +import atexit +import signal import logging import argparse import contextlib @@ -34,10 +38,12 @@ def parse_args(): def setup_logging(): """Setup logging configuration with console and file handlers.""" + # Log brief info to console, but be verbose in a separate file-based log (uploaded as artifact) console_log = logging.StreamHandler(sys.stderr) console_log.setLevel(logging.INFO) debug_log_fobj = gzip.open("atex_debug.log.gz", "wt") + atexit.register(debug_log_fobj.close) file_log = logging.StreamHandler(debug_log_fobj) file_log.setLevel(logging.DEBUG) @@ -49,11 +55,22 @@ def setup_logging(): force=True, ) - return debug_log_fobj + +def setup_signal_handlers(): + """Setup signal handlers for graceful abort.""" + def abort_on_signal(signum, _): + logger.error(f"got signal {signum}, aborting") + raise SystemExit(1) + + signal.signal(signal.SIGTERM, abort_on_signal) + signal.signal(signal.SIGHUP, abort_on_signal) def main(): """Main function to run tests on Testing Farm.""" + setup_logging() + setup_signal_handlers() + args = parse_args() # Variables exported to tests @@ -63,10 +80,6 @@ def main(): } with contextlib.ExitStack() as stack: - # Setup logging - debug_log_fobj = setup_logging() - stack.enter_context(contextlib.closing(debug_log_fobj)) - # Load FMF tests from contest directory fmf_tests = FMFTests( args.contest_dir, @@ -133,9 +146,18 @@ def main(): logger.info("Test execution completed!") - # Log final output locations - logger.info(f"Results written to: {output_results}") - logger.info(f"Test files in: {output_files}") + # Log final output locations + logger.info(f"Results written to: {output_results}") + logger.info(f"Test files in: {output_files}") + + # Read back the compressed JSON results and exit with non-0 if anything failed + with lzma.open(output_results, "rt") as results: + for line in results: + fields = json.loads(line) + # [platform, status, test name, subtest name, files, note] + if fields[1] in ("fail", "error", "infra"): + logger.warning("failures found in the results, exiting with 1") + sys.exit(1) if __name__ == "__main__": diff --git a/tests/submit_results_to_testing_farm.py b/tests/submit_results_to_testing_farm.py index 477035f69297..b9c11d06c7ed 100644 --- a/tests/submit_results_to_testing_farm.py +++ b/tests/submit_results_to_testing_farm.py @@ -20,7 +20,7 @@ def parse_args(): parser = argparse.ArgumentParser(description="Submit TMT test to Testing Farm") parser.add_argument("--repo-url", required=True, help="GitHub repository URL") parser.add_argument("--pr-number", required=True, help="Pull request number") - parser.add_argument("--plan-name", default="/dummy_plan", help="TMT plan name to run") + parser.add_argument("--plan-name", default="/atex_results_plan", help="TMT plan name to run") parser.add_argument("--os", default=None, help="OS to test on (e.g., rhel-9)") parser.add_argument("--arch", default="x86_64", help="Architecture to test on") return parser.parse_args() @@ -82,7 +82,7 @@ def get_html_link(artifacts_url): # {workdir_url}/testing-farm/sanity/execute/results.yaml # as YAML and look for the test name and get its 'data-path' # relative to the /execute/ dir - return f"{workdir_url}/dummy_plan/execute/data/guest/default-0/dummy_test-1/data/index.html?q=TRUE" + return f"{workdir_url}/atex_results_plan/execute/data/guest/default-0/atex_results_test-1/data/index.html?q=status%20IN%20%28%27fail%27%2C%20%27error%27%2C%20%27infra%27%29%20OR%20subtest%20IS%20NULL" def main(): diff --git a/tests/tmt/plans/contest.fmf b/tests/tmt/plans/contest.fmf index 39056e491125..f2c02fcee583 100644 --- a/tests/tmt/plans/contest.fmf +++ b/tests/tmt/plans/contest.fmf @@ -9,122 +9,6 @@ adjust: report: how: html -# -# Hardening via ansible-playbook remediation -# - -/hardening/host-os/ansible/anssi_bp28_high: - discover+: {test: /hardening/host-os/ansible/anssi_bp28_high$} - -/hardening/host-os/ansible/bsi: - discover+: {test: /hardening/host-os/ansible/bsi$} - -/hardening/host-os/ansible/ccn_advanced: - discover+: {test: /hardening/host-os/ansible/ccn_advanced$} - -/hardening/host-os/ansible/cis: - discover+: {test: /hardening/host-os/ansible/cis$} - -/hardening/host-os/ansible/cis_server_l1: - discover+: {test: /hardening/host-os/ansible/cis_server_l1$} - -/hardening/host-os/ansible/cis_workstation_l1: - discover+: {test: /hardening/host-os/ansible/cis_workstation_l1$} - -/hardening/host-os/ansible/cis_workstation_l2: - discover+: {test: /hardening/host-os/ansible/cis_workstation_l2$} - -/hardening/host-os/ansible/cui: - discover+: {test: /hardening/host-os/ansible/cui$} - -/hardening/host-os/ansible/e8: - discover+: {test: /hardening/host-os/ansible/e8$} - -/hardening/host-os/ansible/hipaa: - discover+: {test: /hardening/host-os/ansible/hipaa$} - -/hardening/host-os/ansible/ism_o: - discover+: {test: /hardening/host-os/ansible/ism_o$} - -/hardening/host-os/ansible/ism_o_top_secret: - discover+: {test: /hardening/host-os/ansible/ism_o_top_secret$} - -/hardening/host-os/ansible/ospp: - discover+: {test: /hardening/host-os/ansible/ospp$} - -/hardening/host-os/ansible/pci-dss: - discover+: {test: /hardening/host-os/ansible/pci-dss$} - -/hardening/host-os/ansible/stig: - discover+: {test: /hardening/host-os/ansible/stig$} - -# -# Hardening via oscap xccdf eval --remediate -# - -/hardening/host-os/oscap/anssi_bp28_high: - discover+: {test: /hardening/host-os/oscap/anssi_bp28_high$} - -/hardening/host-os/oscap/bsi: - discover+: {test: /hardening/host-os/oscap/bsi$} - -/hardening/host-os/oscap/ccn_advanced: - discover+: {test: /hardening/host-os/oscap/ccn_advanced$} - -/hardening/host-os/oscap/cis: - discover+: {test: /hardening/host-os/oscap/cis$} - -/hardening/host-os/oscap/cis_server_l1: - discover+: {test: /hardening/host-os/oscap/cis_server_l1$} - -/hardening/host-os/oscap/cis_workstation_l1: - discover+: {test: /hardening/host-os/oscap/cis_workstation_l1$} - -/hardening/host-os/oscap/cis_workstation_l2: - discover+: {test: /hardening/host-os/oscap/cis_workstation_l2$} - -/hardening/host-os/oscap/cui: - discover+: {test: /hardening/host-os/oscap/cui$} - -/hardening/host-os/oscap/e8: - discover+: {test: /hardening/host-os/oscap/e8$} - -/hardening/host-os/oscap/hipaa: - discover+: {test: /hardening/host-os/oscap/hipaa$} - -/hardening/host-os/oscap/ism_o: - discover+: {test: /hardening/host-os/oscap/ism_o$} - -/hardening/host-os/oscap/ism_o_top_secret: - discover+: {test: /hardening/host-os/oscap/ism_o_top_secret$} - -/hardening/host-os/oscap/ospp: - discover+: {test: /hardening/host-os/oscap/ospp$} - -/hardening/host-os/oscap/pci-dss: - discover+: {test: /hardening/host-os/oscap/pci-dss$} - -/hardening/host-os/oscap/stig: - discover+: {test: /hardening/host-os/oscap/stig$} - -# -# Misc smoke/sanity tests -# - -/static-checks: - discover+: - test: /static-checks - exclude: - # exclude here due to the test failing frequently for short periods - # of time, as many websites have temporary availability issues - - /static-checks/html-links - # these always fail, meant for manual review - - /static-checks/diff - # The value of this test is debatable and therefore it should not delay upstream gating. - # Our SCAP datastream is often noncompliant from the start, for example by containing SCE checks. - - /static-checks/nist-validation - - # Fedora specific plan /rpmbuild-ctest-fedora: discover+: {test: /static-checks/rpmbuild-ctest}