[release-4.22] USHIFT-6887: Fix microshift-low-latency for RHEL10

packaging/rpm/microshift.spec

@@ -16,6 +16,13 @@
 # modifying the Go binaries breaks the DWARF debugging
 %global __os_install_post %{_rpmconfigdir}/brp-compress
 
+# TuneD profile directory changed in RHEL 10
+%if 0%{?rhel} >= 10
+%global tuned_profile_dir %{_prefix}/lib/tuned/profiles
+%else
+%global tuned_profile_dir %{_prefix}/lib/tuned
+%endif
+
 # SELinux specifics
 %global selinuxtype targeted
 %define selinux_policyver 3.14.3-67
@@ -438,9 +445,16 @@ install -p -m644 assets/components/multus/release-multus-{x86_64,aarch64}.json %
 install -p -m644 packaging/tuned/microshift-cleanup-kubelet.service %{buildroot}%{_unitdir}/microshift-cleanup-kubelet.service
 
 # low-latency
-install -d -m755 %{buildroot}/%{_prefix}/lib/tuned/microshift-baseline
-install -p -m644 packaging/tuned/profile/tuned.conf %{buildroot}/%{_prefix}/lib/tuned/microshift-baseline/tuned.conf
-install -p -m755 packaging/tuned/profile/script.sh %{buildroot}/%{_prefix}/lib/tuned/microshift-baseline/script.sh
+install -d -m755 %{buildroot}/%{tuned_profile_dir}/microshift-baseline
+install -p -m644 packaging/tuned/profile/tuned.conf %{buildroot}/%{tuned_profile_dir}/microshift-baseline/tuned.conf
+install -p -m755 packaging/tuned/profile/script.sh %{buildroot}/%{tuned_profile_dir}/microshift-baseline/script.sh
+%if 0%{?rhel} < 10
+# TuneD on RHEL 10 looks for profiles in /usr/lib/tuned/profiles/
+# Until we have RHEL10 builders, the profile needs to be symlinked
+# so the package is usable on both RHEL 9 and RHEL 10.
+install -d -m755 %{buildroot}/%{_prefix}/lib/tuned/profiles
+ln -s ../microshift-baseline %{buildroot}/%{_prefix}/lib/tuned/profiles/microshift-baseline
+%endif
 install -d -m755 %{buildroot}%{_sysconfdir}/tuned
 install -p -m644 packaging/tuned/profile/variables.conf %{buildroot}%{_sysconfdir}/tuned/microshift-baseline-variables.conf
 
@@ -744,7 +758,10 @@ fi
 %{_datadir}/microshift/release/release-multus-{x86_64,aarch64}.json
 
 %files low-latency
-%{_prefix}/lib/tuned/microshift-baseline
+%{tuned_profile_dir}/microshift-baseline
+%if 0%{?rhel} < 10
+%{_prefix}/lib/tuned/profiles/microshift-baseline
+%endif
 %config(noreplace) %{_sysconfdir}/tuned/microshift-baseline-variables.conf
 %{_sysconfdir}/crio/crio.conf.d/05-high-performance-runtime.conf
 %{_prefix}/lib/microshift/manifests.d/002-microshift-low-latency/

packaging/tuned/microshift-tuned.py

@@ -115,7 +115,12 @@ def run_command(cmd: list[str], failure_fatal=False) -> tuple[str, bool]:
 
 
 def get_profile_path(profile: str) -> str:
-    paths = [f"/etc/tuned/{profile}", f"/usr/lib/tuned/{profile}"]
+    paths = [
+        f"/etc/tuned/{profile}",
+        f"/etc/tuned/profiles/{profile}",
+        f"/usr/lib/tuned/{profile}",
+        f"/usr/lib/tuned/profiles/{profile}"
+    ]
     for path in paths:
         if os.path.exists(path):
             logging.debug(f"Found profile '{profile}' in '{path}'")

test/image-blueprints-bootc/el10/layer3-periodic/group1/rhel102-bootc-source-tuned.containerfile

@@ -0,0 +1,49 @@
+# hadolint global ignore=DL3059
+FROM localhost/rhel102-bootc-source-optionals:latest
+
+# Enable microshift-tuned service
+RUN systemctl enable microshift-tuned
+
+# Configure MicroShift for low-latency workloads
+RUN cat > /etc/microshift/config.yaml <<'EOF'
+kubelet:
+  cpuManagerPolicy: static
+  cpuManagerPolicyOptions:
+    full-pcpus-only: "true"
+  cpuManagerReconcilePeriod: 5s
+  memoryManagerPolicy: Static
+  topologyManagerPolicy: single-numa-node
+  reservedSystemCPUs: 0-1
+  reservedMemory:
+  - limits:
+      memory: 1100Mi
+    numaNode: 0
+  kubeReserved:
+    memory: 500Mi
+  systemReserved:
+    memory: 500Mi
+  evictionHard:
+    imagefs.available: 15%
+    memory.available: 100Mi
+    nodefs.available: 10%
+    nodefs.inodesFree: 5%
+  evictionPressureTransitionPeriod: 0s
+EOF
+
+# Configure tuned baseline variables
+RUN cat > /etc/tuned/microshift-baseline-variables.conf <<'EOF'
+# Isolated cores should be complementary to kubelet's reserved CPUs.
+# Isolated and reserved CPUs should contain all online CPUs.
+# Core #3 is for testing offlining hence skipped.
+isolated_cores=2,4-5
+hugepages_size=2M
+hugepages=10
+additional_args=test1=on test2=true dummy
+offline_cpu_set=3
+EOF
+
+# Configure MicroShift tuned profile
+RUN cat > /etc/microshift/tuned.yaml <<'EOF'
+profile: microshift-baseline
+reboot_after_apply: True
+EOF

test/image-blueprints-bootc/el9/layer3-periodic/group1/rhel98-bootc-source-tuned.containerfile

@@ -0,0 +1,49 @@
+# hadolint global ignore=DL3059
+FROM localhost/rhel98-bootc-source-optionals:latest
+
+# Enable microshift-tuned service
+RUN systemctl enable microshift-tuned
+
+# Configure MicroShift for low-latency workloads
+RUN cat > /etc/microshift/config.yaml <<'EOF'
+kubelet:
+  cpuManagerPolicy: static
+  cpuManagerPolicyOptions:
+    full-pcpus-only: "true"
+  cpuManagerReconcilePeriod: 5s
+  memoryManagerPolicy: Static
+  topologyManagerPolicy: single-numa-node
+  reservedSystemCPUs: 0-1
+  reservedMemory:
+  - limits:
+      memory: 1100Mi
+    numaNode: 0
+  kubeReserved:
+    memory: 500Mi
+  systemReserved:
+    memory: 500Mi
+  evictionHard:
+    imagefs.available: 15%
+    memory.available: 100Mi
+    nodefs.available: 10%
+    nodefs.inodesFree: 5%
+  evictionPressureTransitionPeriod: 0s
+EOF
+
+# Configure tuned baseline variables
+RUN cat > /etc/tuned/microshift-baseline-variables.conf <<'EOF'
+# Isolated cores should be complementary to kubelet's reserved CPUs.
+# Isolated and reserved CPUs should contain all online CPUs.
+# Core #3 is for testing offlining hence skipped.
+isolated_cores=2,4-5
+hugepages_size=2M
+hugepages=10
+additional_args=test1=on test2=true dummy
+offline_cpu_set=3
+EOF
+
+# Configure MicroShift tuned profile
+RUN cat > /etc/microshift/tuned.yaml <<'EOF'
+profile: microshift-baseline
+reboot_after_apply: True
+EOF

test/scenarios-bootc/el10/periodics/el102-src@low-latency.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+export SKIP_GREENBOOT=true
+export TEST_RANDOMIZATION=none
+
+# Sourced from scenario.sh and uses functions defined there.
+
+start_image="rhel102-bootc-source-tuned"
+
+scenario_create_vms() {
+    exit_if_image_not_found "${start_image}"
+
+    prepare_kickstart host1 kickstart-bootc.ks.template "${start_image}" true
+    launch_vm rhel102-bootc --vm_vcpus 6
+}
+
+scenario_remove_vms() {
+    exit_if_image_not_found "${start_image}"
+
+    remove_vm host1
+}
+
+scenario_run_tests() {
+    exit_if_image_not_found "${start_image}"
+
+    # Should not be run immediately after creating VM because of
+    # microshift-tuned rebooting the node to activate the profile.
+    local -r start_time=$(date +%s)
+    while true; do
+        boot_num=$(run_command_on_vm host1 "sudo journalctl --list-boots --quiet | wc -l" || true)
+        boot_num="${boot_num%$'\r'*}"
+        if [[ "${boot_num}" -ge 2 ]]; then
+            break
+        fi
+        if [ $(( $(date +%s) - start_time )) -gt 60 ]; then
+            echo "Timed out waiting for VM having 2 boots"
+            exit 1
+        fi
+        sleep 5
+    done
+
+    # --exitonfailure because tests within suites are meant to be ordered,
+    # so don't advance to next test if current failed.
+
+    run_tests host1 \
+        --exitonfailure \
+        suites/tuned/microshift-tuned.robot \
+        suites/tuned/workload-partitioning.robot \
+        suites/tuned/uncore-cache.robot
+}

test/scenarios-bootc/el9/periodics/el98-src@low-latency.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+export SKIP_GREENBOOT=true
+export TEST_RANDOMIZATION=none
+
+# Sourced from scenario.sh and uses functions defined there.
+
+start_image="rhel98-bootc-source-tuned"
+
+scenario_create_vms() {
+    exit_if_image_not_found "${start_image}"
+
+    prepare_kickstart host1 kickstart-bootc.ks.template "${start_image}" true
+    launch_vm rhel98-bootc --vm_vcpus 6
+}
+
+scenario_remove_vms() {
+    exit_if_image_not_found "${start_image}"
+
+    remove_vm host1
+}
+
+scenario_run_tests() {
+    exit_if_image_not_found "${start_image}"
+
+    # Should not be run immediately after creating VM because of
+    # microshift-tuned rebooting the node to activate the profile.
+    local -r start_time=$(date +%s)
+    while true; do
+        boot_num=$(run_command_on_vm host1 "sudo journalctl --list-boots --quiet | wc -l" || true)
+        boot_num="${boot_num%$'\r'*}"
+        if [[ "${boot_num}" -ge 2 ]]; then
+            break
+        fi
+        if [ $(( $(date +%s) - start_time )) -gt 60 ]; then
+            echo "Timed out waiting for VM having 2 boots"
+            exit 1
+        fi
+        sleep 5
+    done
+
+    # --exitonfailure because tests within suites are meant to be ordered,
+    # so don't advance to next test if current failed.
+
+    run_tests host1 \
+        --exitonfailure \
+        suites/tuned/microshift-tuned.robot \
+        suites/tuned/workload-partitioning.robot \
+        suites/tuned/uncore-cache.robot
+}

test/suites/tuned/workload-partitioning.robot

@@ -64,6 +64,7 @@ Teardown For Workload Partitioning
     Remove Drop In MicroShift Config    10-kubelet
     Systemctl    restart    crio.service
     Restart MicroShift
+    Wait For MicroShift Healthcheck Success
 
 Configure Kubelet For Workload Partitioning
     [Documentation]    configure microshift with kubelet CPU configuration
@@ -252,6 +253,7 @@ Cleanup And Create NS
     Cleanup MicroShift    --all    --keep-images
     Remove Files    ${KUBELET_CPU_STATE_FILE}
     Restart MicroShift
+    Wait For MicroShift Healthcheck Success
     ${ns}=    Create Unique Namespace
     VAR    ${NAMESPACE}=    ${ns}    scope=SUITE