From d1bd76afac52932662c0935c22219bdd8004c0ad Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 20 Feb 2024 14:45:56 -0500 Subject: [PATCH 1/7] common.yaml: separate postprocessing steps more I think it's confusing when a single postprocessing item actually does multiple disparate things. Let's try to split them up to make it clearer. While we're here, make the indentation consistent. This patch should have no functional effect. Best viewed with whitespace changes ignored. --- common.yaml | 121 +++++++++++++++++++++++++++++----------------------- 1 file changed, 67 insertions(+), 54 deletions(-) diff --git a/common.yaml b/common.yaml index d90ec08e9..fbf828a65 100644 --- a/common.yaml +++ b/common.yaml @@ -36,35 +36,41 @@ conditional-include: documentation: false postprocess: + # TEMPORARY: Create /etc/vmware-tools/tools.conf to ensure RHCOS shows up properly in VMWare + # See https://jira.coreos.com/browse/RHCOS-258 - | - #!/usr/bin/env bash - set -xeo pipefail + #!/usr/bin/env bash + set -xeo pipefail + if [ "$(uname -m)" == "x86_64" ]; then + cat > /etc/vmware-tools/tools.conf <<'EOF' + [guestosinfo] + short-name = rhel8-64 + EOF + fi - # TEMPORARY: Create /etc/vmware-tools/tools.conf to ensure RHCOS shows up properly in VMWare - # See https://jira.coreos.com/browse/RHCOS-258 - if [ "$(uname -m)" == "x86_64" ]; then - cat > /etc/vmware-tools/tools.conf <<'EOF' - [guestosinfo] - short-name = rhel8-64 - EOF - fi + # TEMPORARY: Remove /usr/etc/zkey/ group write permissions + # See https://github.com/openshift/os/issues/1209 + - | + #!/usr/bin/env bash + set -xeo pipefail + if [ "$(uname -m)" == "s390x" ]; then + chmod -R g-w /usr/etc/zkey + fi - # TEMPORARY: Remove /usr/etc/zkey/ group write permissions - # See https://github.com/openshift/os/issues/1209 - if [ "$(uname -m)" == "s390x" ]; then - chmod -R g-w /usr/etc/zkey - fi + # Nuke network.service from orbit + # https://github.com/openshift/os/issues/117 + - | + #!/usr/bin/env bash + set -xeo pipefail + rm -rf /etc/rc.d/init.d/network /etc/rc.d/rc*.d/*network - # Nuke network.service from orbit - # https://github.com/openshift/os/issues/117 - rm -rf /etc/rc.d/init.d/network /etc/rc.d/rc*.d/*network + # We're not using resolved yet + rm -f /usr/lib/systemd/system/systemd-resolved.service - # We're not using resolved yet - rm -f /usr/lib/systemd/system/systemd-resolved.service + # manually modify SELinux booleans that are needed for OCP use cases - | #!/usr/bin/env bash set -xeuo pipefail - # manually modify SELinux booleans that are needed for OCP use cases semanage boolean --modify --on container_use_cephfs # RHBZ#1694045 semanage boolean --modify --on virt_use_samba # RHBZ#1754825 @@ -139,45 +145,52 @@ postprocess: done done - | - #!/usr/bin/env bash - set -xeo pipefail + #!/usr/bin/env bash + set -xeo pipefail - # FIXME: Why is this only broken here? NM isn't removing the link? - sed -i '/etc.resolv/d' /usr/lib/tmpfiles.d/etc.conf + # FIXME: Why is this only broken here? NM isn't removing the link? + sed -i '/etc.resolv/d' /usr/lib/tmpfiles.d/etc.conf - # crio should stop hardcoding things in their config file! - # We are apparently somehow pulling in a conmon override in RHCOS - # that contains /usr/libexec/crio/conmon - WHY? - # sed -i '/conmon.*=/d' /etc/crio/crio.conf - # Oh right but the MCO overrides that too so... - mkdir -p /usr/libexec/crio - ln -sr /usr/bin/conmon /usr/libexec/crio/conmon + - | + #!/usr/bin/env bash + set -xeo pipefail + # crio should stop hardcoding things in their config file! + # We are apparently somehow pulling in a conmon override in RHCOS + # that contains /usr/libexec/crio/conmon - WHY? + # sed -i '/conmon.*=/d' /etc/crio/crio.conf + # Oh right but the MCO overrides that too so... + mkdir -p /usr/libexec/crio + ln -sr /usr/bin/conmon /usr/libexec/crio/conmon - # Enable tmp-on-tmpfs by default because we don't want to have things leak - # across reboots, it increases alignment with FCOS, and also fixes the - # Live ISO. First, verify that RHEL is still disabling. - grep -q '# RHEL-only: Disable /tmp on tmpfs' /usr/lib/systemd/system/basic.target - echo '# RHCOS-only: we follow the Fedora/upstream default' >> /usr/lib/systemd/system/basic.target - echo 'Wants=tmp.mount' >> /usr/lib/systemd/system/basic.target - | - #!/usr/bin/env bash - set -xeo pipefail - # See https://issues.redhat.com/browse/LOG-3117 - # Something changed between rhel8 and rhel9 to not generate this by default - # but we have containers that expect it to be mounted so for now let's continue - # generating it. - ln -sr /usr/share/zoneinfo/UTC /etc/localtime + #!/usr/bin/env bash + set -xeuo pipefail + # Enable tmp-on-tmpfs by default because we don't want to have things leak + # across reboots, it increases alignment with FCOS, and also fixes the + # Live ISO. First, verify that RHEL is still disabling. + grep -q '# RHEL-only: Disable /tmp on tmpfs' /usr/lib/systemd/system/basic.target + echo '# RHCOS-only: we follow the Fedora/upstream default' >> /usr/lib/systemd/system/basic.target + echo 'Wants=tmp.mount' >> /usr/lib/systemd/system/basic.target + - | - #!/usr/bin/env bash - set -xeo pipefail - # Add the hugetlbfs group to the openvswitch user if the openvswitch-hugetlbfs.conf - # sysusers fragment exists. The usermod used to happen in the RPM scriptlets but - # that stopped working in the sysusers conversion. We should be able to drop this - # when a bug gets fixed in systemd: https://github.com/openshift/os/issues/1274#issuecomment-1605507390 - if [ -f /usr/lib/sysusers.d/openvswitch-hugetlbfs.conf ]; then - usermod -a -G hugetlbfs openvswitch - fi + #!/usr/bin/env bash + set -xeo pipefail + # See https://issues.redhat.com/browse/LOG-3117 + # Something changed between rhel8 and rhel9 to not generate this by default + # but we have containers that expect it to be mounted so for now let's continue + # generating it. + ln -sr /usr/share/zoneinfo/UTC /etc/localtime + - | + #!/usr/bin/env bash + set -xeo pipefail + # Add the hugetlbfs group to the openvswitch user if the openvswitch-hugetlbfs.conf + # sysusers fragment exists. The usermod used to happen in the RPM scriptlets but + # that stopped working in the sysusers conversion. We should be able to drop this + # when a bug gets fixed in systemd: https://github.com/openshift/os/issues/1274#issuecomment-1605507390 + if [ -f /usr/lib/sysusers.d/openvswitch-hugetlbfs.conf ]; then + usermod -a -G hugetlbfs openvswitch + fi remove-files: # We don't ship man(1) or info(1) From 13a9405e96947e0047b8582fd2a85ea65a00c583 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 20 Feb 2024 15:03:12 -0500 Subject: [PATCH 2/7] manifests: move OCP-related postprocessing to `packages-openshift.yaml` As prep for #799, let's better split the postprocessing steps that are related to OCP from those that have tighter binding to RHEL proper. This should have no visible effect. --- common.yaml | 49 --------------------------------- manifest-c9s.yaml | 7 ----- manifest-rhel-9.4.yaml | 7 ----- packages-openshift.yaml | 60 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 63 deletions(-) diff --git a/common.yaml b/common.yaml index fbf828a65..ea65292c0 100644 --- a/common.yaml +++ b/common.yaml @@ -67,33 +67,6 @@ postprocess: # We're not using resolved yet rm -f /usr/lib/systemd/system/systemd-resolved.service - # manually modify SELinux booleans that are needed for OCP use cases - - | - #!/usr/bin/env bash - set -xeuo pipefail - semanage boolean --modify --on container_use_cephfs # RHBZ#1694045 - semanage boolean --modify --on virt_use_samba # RHBZ#1754825 - - # https://gitlab.cee.redhat.com/coreos/redhat-coreos/merge_requests/812 - # https://bugzilla.redhat.com/show_bug.cgi?id=1796537 - - | - #!/usr/bin/bash - mkdir -p /usr/share/containers/oci/hooks.d - - # This is part of e.g. fedora-repos in Fedora; we now want to include it by default - # so that the MCO can use it by default and not trip over SELinux issues trying - # to create it. - - | - #!/usr/bin/bash - mkdir -p /etc/yum.repos.d - - # These enable librhsm which enables host subscriptions to work in containers - # https://github.com/rpm-software-management/librhsm/blob/fcd972cbe7c8a3907ba9f091cd082b1090231492/rhsm/rhsm-context.c#L30 - - | - #!/usr/bin/bash - ln -sr /run/secrets/etc-pki-entitlement /etc/pki/entitlement-host - ln -sr /run/secrets/rhsm /etc/rhsm-host - # This updates the PAM configuration to reference all of the SSSD modules. # Removes the `authselect` binary afterwards since `authselect` does not play well with `nss-altfiles` # (https://github.com/pbrezina/authselect/issues/48). @@ -151,17 +124,6 @@ postprocess: # FIXME: Why is this only broken here? NM isn't removing the link? sed -i '/etc.resolv/d' /usr/lib/tmpfiles.d/etc.conf - - | - #!/usr/bin/env bash - set -xeo pipefail - # crio should stop hardcoding things in their config file! - # We are apparently somehow pulling in a conmon override in RHCOS - # that contains /usr/libexec/crio/conmon - WHY? - # sed -i '/conmon.*=/d' /etc/crio/crio.conf - # Oh right but the MCO overrides that too so... - mkdir -p /usr/libexec/crio - ln -sr /usr/bin/conmon /usr/libexec/crio/conmon - - | #!/usr/bin/env bash set -xeuo pipefail @@ -181,17 +143,6 @@ postprocess: # generating it. ln -sr /usr/share/zoneinfo/UTC /etc/localtime - - | - #!/usr/bin/env bash - set -xeo pipefail - # Add the hugetlbfs group to the openvswitch user if the openvswitch-hugetlbfs.conf - # sysusers fragment exists. The usermod used to happen in the RPM scriptlets but - # that stopped working in the sysusers conversion. We should be able to drop this - # when a bug gets fixed in systemd: https://github.com/openshift/os/issues/1274#issuecomment-1605507390 - if [ -f /usr/lib/sysusers.d/openvswitch-hugetlbfs.conf ]; then - usermod -a -G hugetlbfs openvswitch - fi - remove-files: # We don't ship man(1) or info(1) - usr/share/info diff --git a/manifest-c9s.yaml b/manifest-c9s.yaml index e828ae4cc..bb0d240aa 100644 --- a/manifest-c9s.yaml +++ b/manifest-c9s.yaml @@ -112,13 +112,6 @@ postprocess: --- EOF - - | - #!/usr/bin/env bash - set -xeo pipefail - # We need to work in disconnected environments by default, and default-enabled - # repos will be attempted to be fetched by rpm-ostree when doing node-local - # kernel overrides today for e.g. kernel-rt. - for x in /etc/yum.repos.d/*.repo; do sed -i -e s,enabled=1,enabled=0, $x; done # Packages that are only in SCOS and not in RHCOS or that have special # constraints that do not apply to RHCOS diff --git a/manifest-rhel-9.4.yaml b/manifest-rhel-9.4.yaml index 1f0fcdd8c..f86db050d 100644 --- a/manifest-rhel-9.4.yaml +++ b/manifest-rhel-9.4.yaml @@ -108,13 +108,6 @@ postprocess: --- EOF - - | - #!/usr/bin/env bash - set -xeo pipefail - # We need to work in disconnected environments by default, and default-enabled - # repos will be attempted to be fetched by rpm-ostree when doing node-local - # kernel overrides today for e.g. kernel-rt. - for x in /etc/yum.repos.d/*.repo; do sed -i -e s,enabled=1,enabled=0, $x; done # Packages that are only in RHCOS and not in SCOS or that have special # constraints that do not apply to SCOS diff --git a/packages-openshift.yaml b/packages-openshift.yaml index fece50b5b..74c4bff37 100644 --- a/packages-openshift.yaml +++ b/packages-openshift.yaml @@ -10,3 +10,63 @@ packages: - ose-aws-ecr-image-credential-provider - ose-azure-acr-image-credential-provider - ose-gcp-gcr-image-credential-provider + +postprocess: + # This is part of e.g. fedora-repos in Fedora; we now want to include it by default + # so that the MCO can use it by default and not trip over SELinux issues trying + # to create it. + - | + #!/usr/bin/bash + set -euo pipefail + mkdir -p /etc/yum.repos.d + + # If there *are* repos built-in (e.g. c9s), disable them. + # We need to work in disconnected environments by default, and default-enabled + # repos will be attempted to be fetched by rpm-ostree when doing node-local + # kernel overrides today for e.g. kernel-rt. + for x in $(find /etc/yum.repos.d/ -name '*.repo'); do + sed -i -e s,enabled=1,enabled=0, $x + done + + # These enable librhsm which enables host subscriptions to work in containers + # https://github.com/rpm-software-management/librhsm/blob/fcd972cbe7c8a3907ba9f091cd082b1090231492/rhsm/rhsm-context.c#L30 + - | + #!/usr/bin/bash + set -euo pipefail + ln -sr /run/secrets/etc-pki-entitlement /etc/pki/entitlement-host + ln -sr /run/secrets/rhsm /etc/rhsm-host + + - | + #!/usr/bin/env bash + set -xeuo pipefail + # manually modify SELinux booleans that are needed for OCP use cases + semanage boolean --modify --on container_use_cephfs # RHBZ#1694045 + semanage boolean --modify --on virt_use_samba # RHBZ#1754825 + + # https://gitlab.cee.redhat.com/coreos/redhat-coreos/merge_requests/812 + # https://bugzilla.redhat.com/show_bug.cgi?id=1796537 + - | + #!/usr/bin/bash + mkdir -p /usr/share/containers/oci/hooks.d + + - | + #!/usr/bin/env bash + set -xeo pipefail + # Add the hugetlbfs group to the openvswitch user if the openvswitch-hugetlbfs.conf + # sysusers fragment exists. The usermod used to happen in the RPM scriptlets but + # that stopped working in the sysusers conversion. We should be able to drop this + # when a bug gets fixed in systemd: https://github.com/openshift/os/issues/1274#issuecomment-1605507390 + if [ -f /usr/lib/sysusers.d/openvswitch-hugetlbfs.conf ]; then + usermod -a -G hugetlbfs openvswitch + fi + + - | + #!/usr/bin/env bash + set -xeuo pipefail + # crio should stop hardcoding things in their config file! + # We are apparently somehow pulling in a conmon override in RHCOS + # that contains /usr/libexec/crio/conmon - WHY? + # sed -i '/conmon.*=/d' /etc/crio/crio.conf + # Oh right but the MCO overrides that too so... + mkdir -p /usr/libexec/crio + ln -sr /usr/bin/conmon /usr/libexec/crio/conmon From 4af54f712dcfee35c7cd9dd80222e5ef191c3476 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 20 Feb 2024 15:07:48 -0500 Subject: [PATCH 3/7] common.yaml: drop outdated comments --- common.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/common.yaml b/common.yaml index ea65292c0..87fbfa7be 100644 --- a/common.yaml +++ b/common.yaml @@ -197,7 +197,6 @@ packages: - bootupd # Containers - containernetworking-plugins - # Pinned due to cosa on Fedora not honoring RHEL 8 modules as expected - container-selinux # Needed for newer rpm-ostree - "'skopeo >= 2:1.7.0'" @@ -226,7 +225,6 @@ packages: - qemu-guest-agent # passwd was obsoleted by shadow-utils in F40+, but still needed here - passwd - # BELOW HERE ARE PACKAGES NOT IN RHEL # Gluster - Used for Openshift e2e gluster testcases # Reverts https://gitlab.cee.redhat.com/coreos/redhat-coreos/merge_requests/367 and add it for all arches - glusterfs-fuse From 3a602004c12ec96f1ebce3c51ee34dd943720138 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 20 Feb 2024 15:08:07 -0500 Subject: [PATCH 4/7] common.yaml: explicitly list policycoreutils-python-utils We were pulling this in transitively, but we do depend on it in postprocessing steps (for semanage). Let's make it explicit. --- common.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common.yaml b/common.yaml index 87fbfa7be..25ae04af7 100644 --- a/common.yaml +++ b/common.yaml @@ -242,6 +242,8 @@ packages: - WALinuxAgent-udev # Provide fips-mode-setup which is needed by rhcos-fips.sh - crypto-policies-scripts + # For semanage + - policycoreutils-python-utils packages-x86_64: # Temporary add of open-vm-tools. Should be removed when containerized From 1841132d3cadf406a694b722df6e3d0fb9bfb456 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 20 Feb 2024 15:09:23 -0500 Subject: [PATCH 5/7] tests/env-godebug: simplify initramfs finding Just get at the initramfs using a glob instead of trying to parse the BLS. This will work regardless of how the BLS entry is named but assumes that there is only one BLS entry (which should always be the case for these tests). --- tests/kola/files/env-godebug | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/kola/files/env-godebug b/tests/kola/files/env-godebug index 8f902b0bc..60733720e 100755 --- a/tests/kola/files/env-godebug +++ b/tests/kola/files/env-godebug @@ -10,9 +10,7 @@ set -xeuo pipefail . $KOLA_EXT_DATA/commonlib.sh -source /etc/os-release -ostree_conf="/boot/loader.1/entries/ostree-1-${ID}.conf" -initramfs=/boot$(grep initrd ${ostree_conf} | sed 's/initrd //g') +initramfs=$(ls /boot/ostree/*/initramfs-*) conf="etc/systemd/system.conf.d/10-default-env-godebug.conf" tempd=$(mktemp -d) # unpack 10-default-env-godebug.conf from initramfs file From 0f34823319c350d3c8dda4d0715790017574e4ff Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 7 Mar 2024 15:30:19 -0500 Subject: [PATCH 6/7] tests: tag OpenShift-specific tests with `openshift` This is part of #1445. Those tests are all actually testing OCP components. In the new model, they should be run against an OCP layered image instead. Add a tag on them so that we'll be able to run them separately. --- tests/kola/files/openvswitch-hugetlbfs-groups | 1 + tests/kola/version/rhel-matches-rhcos-build | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/kola/files/openvswitch-hugetlbfs-groups b/tests/kola/files/openvswitch-hugetlbfs-groups index cf7c395b8..06f8a7243 100755 --- a/tests/kola/files/openvswitch-hugetlbfs-groups +++ b/tests/kola/files/openvswitch-hugetlbfs-groups @@ -3,6 +3,7 @@ ## exclusive: false ## architectures: "x86_64 ppc64le" ## description: Verify openvswitch user is in the hugetlbfs group. +## tags: openshift set -xeuo pipefail diff --git a/tests/kola/version/rhel-matches-rhcos-build b/tests/kola/version/rhel-matches-rhcos-build index 8116a9f24..55e86e4a5 100755 --- a/tests/kola/version/rhel-matches-rhcos-build +++ b/tests/kola/version/rhel-matches-rhcos-build @@ -3,6 +3,7 @@ ## exclusive: false ## description: Verify that the OS version (C9S, RHEL 9.x) matches ## the version stored in /etc/os-release. +## tags: openshift set -xeuo pipefail From 0abce5c6734f06f7cda5be182018755a14397562 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 20 Feb 2024 15:34:22 -0500 Subject: [PATCH 7/7] manifests: move bootc up to common.yaml It's in all the development streams we care about now so just simplify the manifests. No functional change. --- common.yaml | 2 ++ manifest-c9s.yaml | 1 - manifest-rhel-9.4.yaml | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common.yaml b/common.yaml index 25ae04af7..0e42b663c 100644 --- a/common.yaml +++ b/common.yaml @@ -210,6 +210,8 @@ packages: # Common tools used by scripts and admins interactively - rsync tmux - nmap-ncat strace + # Future update manager on the client-side + - bootc # Editors - nano # Red Hat CA certs diff --git a/manifest-c9s.yaml b/manifest-c9s.yaml index bb0d240aa..fa9a7ca7b 100644 --- a/manifest-c9s.yaml +++ b/manifest-c9s.yaml @@ -123,7 +123,6 @@ packages: - centos-release-cloud-common - centos-release-nfv-common - centos-release-virt-common - - bootc # Packages pinned to specific repos in SCOS 9 repo-packages: diff --git a/manifest-rhel-9.4.yaml b/manifest-rhel-9.4.yaml index f86db050d..6ecd224ea 100644 --- a/manifest-rhel-9.4.yaml +++ b/manifest-rhel-9.4.yaml @@ -115,7 +115,6 @@ packages: # We include the generic release package and tweak the os-release info in a # post-proces script - centos-release - - bootc # Packages pinned to specific repos in SCOS 9 repo-packages: