[release-4.22] OCPBUGS-84533: RHEL 10 RPM-based installation testing in CI

test/bin/ci_phase_iso_build.sh

@@ -137,8 +137,7 @@ run_bootc_image_build() {
 
             if [[ "${os}" == "el10" ]]; then
                 # Build el9 images for upgrade tests
-                $(dry_run) bash -x ./bin/build_bootc_images.sh -l ./image-blueprints-bootc/el9/layer1-base
-                $(dry_run) bash -x ./bin/build_bootc_images.sh -l ./image-blueprints-bootc/el9/layer2-presubmit
+                $(dry_run) bash -x ./bin/build_bootc_images.sh -l ./image-blueprints-bootc/el10/layer5-upgrade
             fi
 
             if [[ "${CI_JOB_NAME}" =~ .*periodic.* ]]; then

test/bin/pyutils/build_bootc_images.py

@@ -34,6 +34,7 @@
 # Switch to quay.io/centos-bootc/bootc-image-builder:latest if any new upstream
 # features are required
 BIB_IMAGE = "registry.redhat.io/rhel9/bootc-image-builder:latest"
+IBC_IMAGE = "ghcr.io/osbuild/image-builder-cli:latest"
 GOMPLATE = common.get_env_var('GOMPLATE')
 MIRROR_REGISTRY = common.get_env_var('MIRROR_REGISTRY_URL')
 FORCE_REBUILD = False
@@ -46,18 +47,19 @@ def cleanup_atexit(dry_run):
         common.print_msg(f"Terminating {pid} PID")
         common.terminate_process(pid)
 
-    # Terminate running bootc image builder containers
-    podman_args = [
-        "sudo", "podman", "ps",
-        "--filter", f"ancestor={BIB_IMAGE}",
-        "--format", "{{.ID}}"
-    ]
-    cids = common.run_command_in_shell(podman_args, dry_run)
-    if cids:
-        # Make sure the ids are normalized in a single line
-        cids = re.sub(r'\s+', ' ', cids)
-        common.print_msg(f"Terminating '{cids}' container(s)")
-        common.run_command_in_shell(["sudo", "podman", "stop", cids], dry_run)
+    # Terminate running image builder containers
+    for builder_image in [BIB_IMAGE, IBC_IMAGE]:
+        podman_args = [
+            "sudo", "podman", "ps",
+            "--filter", f"ancestor={builder_image}",
+            "--format", "{{.ID}}"
+        ]
+        cids = common.run_command_in_shell(podman_args, dry_run)
+        if cids:
+            # Make sure the ids are normalized in a single line
+            cids = re.sub(r'\s+', ' ', cids)
+            common.print_msg(f"Terminating '{cids}' container(s)")
+            common.run_command_in_shell(["sudo", "podman", "stop", cids], dry_run)
 
 
 def find_latest_rpm(repo_path, version=""):
@@ -430,6 +432,76 @@ def should_skip(file):
         os.rename(f"{bf_outdir}/bootiso/install.iso", bf_targetiso)
 
 
+def process_image_installer(groupdir, installerfile, dry_run):
+    ii_path, ii_outname, ii_outdir, ii_logfile = get_process_file_names(
+        groupdir, installerfile, BOOTC_ISO_DIR)
+    ii_targetiso = os.path.join(VM_DISK_BASEDIR, f"{ii_outname}.iso")
+
+    def should_skip(file):
+        if FORCE_REBUILD:
+            common.print_msg(f"Forcing rebuild of '{file}'")
+            return False
+        if not os.path.exists(file):
+            return False
+        common.print_msg(f"The '{file}' already exists, skipping")
+        return True
+
+    if should_skip(ii_targetiso):
+        common.record_junit(ii_path, "process-image-installer", "SKIPPED")
+        return
+
+    os.makedirs(ii_outdir, exist_ok=True)
+    os.makedirs(VM_DISK_BASEDIR, exist_ok=True)
+    ii_outfile = os.path.join(BOOTC_IMAGE_DIR, installerfile)
+    run_template_cmd(ii_path, ii_outfile, dry_run)
+    if not dry_run:
+        if not common.file_has_valid_lines(ii_outfile):
+            common.print_msg(f"Skipping an empty {installerfile} file")
+            return
+
+    common.print_msg(f"Processing {installerfile} with logs in {ii_logfile}")
+    start_process_image_installer = time.time()
+    try:
+        with open(ii_logfile, 'w') as logfile:
+            ii_distro = common.read_file_valid_lines(ii_outfile).strip()
+
+            build_args = [
+                "sudo", "podman", "run",
+                "--rm", "-i", "--privileged",
+                "--network", "host",
+                "--pull=newer",
+                "--security-opt", "label=type:unconfined_t",
+                "-v", f"{ii_outdir}:/output",
+            ]
+            if os.path.isdir("/etc/pki/entitlement"):
+                build_args += [
+                    "-v", "/etc/pki/entitlement:/etc/pki/entitlement:ro",
+                    "-v", "/etc/rhsm:/etc/rhsm:ro",
+                ]
+            build_args += [
+                IBC_IMAGE,
+                "build", "--distro", ii_distro,
+                "image-installer"
+            ]
+            start = time.time()
+            common.retry_on_exception(3, common.run_command_in_shell, build_args, dry_run, logfile, logfile)
+            common.record_junit(ii_path, "build-installer-image", "OK", start)
+    except Exception:
+        common.record_junit(ii_path, "process-image-installer", "FAILED", start_process_image_installer, log_filepath=ii_logfile)
+        raise
+    finally:
+        common.run_command(["sed", f"s/^/{ii_outname}: /", ii_logfile], dry_run)
+
+    if not dry_run:
+        common.run_command(
+            ["sudo", "chown", "-R", f"{getpass.getuser()}.", ii_outdir],
+            dry_run)
+        iso_candidates = glob.glob(f"{ii_outdir}/**/*.iso", recursive=True)
+        if not iso_candidates:
+            raise Exception(f"No ISO found in {ii_outdir}")
+        os.rename(iso_candidates[0], ii_targetiso)
+
+
 def process_container_encapsulate(groupdir, containerfile, dry_run):
     ce_path, ce_outname, _, ce_logfile = get_process_file_names(
         groupdir, containerfile, BOOTC_IMAGE_DIR)
@@ -546,6 +618,11 @@ def process_group(groupdir, build_type, pattern="*", dry_run=False):
                         common.print_msg(f"Skipping '{file}' due to '{build_type}' filter")
                         continue
                     futures.append(executor.submit(process_image_bootc, groupdir, file, dry_run))
+                elif file.endswith(".image-installer"):
+                    if build_type and build_type != "image-installer":
+                        common.print_msg(f"Skipping '{file}' due to '{build_type}' filter")
+                        continue
+                    futures.append(executor.submit(process_image_installer, groupdir, file, dry_run))
                 elif file.endswith(".container-encapsulate"):
                     if build_type and build_type != "container-encapsulate":
                         common.print_msg(f"Skipping '{file}' due to '{build_type}' filter")
@@ -580,7 +657,7 @@ def main():
     parser.add_argument("-E", "--no-extract-images", action="store_true", help="Skip container image extraction.")
     parser.add_argument("-X", "--skip-all-builds", action="store_true", help="Skip all image builds.")
     parser.add_argument("-b", "--build-type",
-                        choices=["image-bootc", "containerfile", "container-encapsulate"],
+                        choices=["image-bootc", "image-installer", "containerfile", "container-encapsulate"],
                         help="Only build images of the specified type.")
     dirgroup = parser.add_mutually_exclusive_group(required=False)
     dirgroup.add_argument("-l", "--layer-dir", action="append", default=[], help="Path to the layer directory to process. Can be specified multiple times.")

test/image-blueprints-bootc/el10/layer1-base/group2/rhel102-installer.image-installer

@@ -0,0 +1,4 @@
+
+# TODO: Replace this by a RHEL 10.2 image when its RPM repositories are released.
+# rhel-10.2
+rhel-10.1

test/image-blueprints-bootc/el10/layer5-upgrade/group1/rhel96-bootc.image-bootc

@@ -0,0 +1 @@
+../../../el9/layer1-base/group2/rhel96-bootc.image-bootc

test/image-blueprints-bootc/el10/layer5-upgrade/group1/rhel96-test-agent.containerfile

@@ -0,0 +1 @@
+../../../el9/layer1-base/group1/rhel96-test-agent.containerfile

test/image-blueprints-bootc/el10/layer5-upgrade/group1/rhel98-bootc.image-bootc

@@ -0,0 +1 @@
+../../../el9/layer1-base/group2/rhel98-bootc.image-bootc

test/image-blueprints-bootc/el10/layer5-upgrade/group1/rhel98-test-agent.containerfile

@@ -0,0 +1 @@
+../../../el9/layer1-base/group1/rhel98-test-agent.containerfile

test/image-blueprints-bootc/el10/layer5-upgrade/group2/rhel96-bootc-prel.containerfile

@@ -0,0 +1 @@
+../../../el9/layer1-base/group2/rhel96-bootc-prel.containerfile

test/image-blueprints-bootc/el10/layer5-upgrade/group2/rhel96-bootc-yminus2.containerfile

@@ -0,0 +1 @@
+../../../el9/layer1-base/group2/rhel96-bootc-yminus2.containerfile

test/image-blueprints-bootc/el10/layer5-upgrade/group2/rhel98-bootc-source.containerfile

@@ -0,0 +1 @@
+../../../el9/layer2-presubmit/group1/rhel98-bootc-source.containerfile

test/image-blueprints-bootc/el9/layer1-base/group2/rhel98-installer.image-installer

@@ -0,0 +1,4 @@
+
+# TODO: Replace this by a RHEL 9.8 image when its RPM repositories are released.
+# rhel-9.8
+rhel-9.7

test/scenarios-bootc/el10/presubmits/el102-src@rpm-install.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+
+# Sourced from scenario.sh and uses functions defined there.
+
+# The RPM-based image used to create the VM for this test does not
+# include MicroShift or greenboot, so tell the framework not to wait
+# for greenboot to finish when creating the VM.
+export SKIP_GREENBOOT=true
+
+# NOTE: Unlike most suites, these tests rely on being run IN ORDER to
+# ensure the host is in a good state at the start of each test. We
+# could have separated them and run them as separate scenarios, but
+# did not want to spend the resources on a new VM.
+export TEST_RANDOMIZATION=none
+
+configure_microshift_mirror() {
+    local -r repo=$1
+
+    # `repo` might be empty if we install microshift from rhocp
+    if [[ -z "${repo}" ]] ; then
+        return
+    fi
+
+    # `repo` might be an enabled repo from a released version instead
+    # of a mirror.
+    if [[ ! "${repo}" =~ ^http ]]; then
+        return
+    fi
+
+    local -r tmp_file=$(mktemp)
+    tee "${tmp_file}" >/dev/null <<EOF
+[microshift-mirror-rpms]
+name=MicroShift Mirror
+baseurl=${repo}
+enabled=1
+gpgcheck=0
+skip_if_unavailable=0
+EOF
+    copy_file_to_vm host1 "${tmp_file}" "${tmp_file}"
+    run_command_on_vm host1 "sudo cp ${tmp_file} /etc/yum.repos.d/microshift-mirror-rpms.repo"
+    rm -f "${tmp_file}"
+}
+
+# On RHEL 10, rhocp and fast-datapath repos are not available via
+# subscription-manager. Create repo files pointing to the RHEL 9 CDN
+# using entitlement certificates as a workaround.
+configure_cdn_repo() {
+    local -r repo_id=$1
+    local -r repo_name=$2
+    local -r baseurl=$3
+
+    local -r cert=$(run_command_on_vm host1 "ls /etc/pki/entitlement/[0-9]*.pem | grep -v '\-key.pem' | head -n1")
+    local -r key=$(run_command_on_vm host1 "ls /etc/pki/entitlement/[0-9]*-key.pem | head -n1")
+    local -r tmp_file=$(mktemp)
+
+    tee "${tmp_file}" >/dev/null <<EOF
+[${repo_id}]
+name=${repo_name}
+baseurl=${baseurl}
+enabled=1
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
+sslverify=1
+sslcacert=/etc/rhsm/ca/redhat-uep.pem
+sslclientcert=${cert}
+sslclientkey=${key}
+EOF
+    copy_file_to_vm host1 "${tmp_file}" "${tmp_file}"
+    run_command_on_vm host1 "sudo cp ${tmp_file} /etc/yum.repos.d/${repo_id}.repo"
+    rm -f "${tmp_file}"
+}
+
+configure_rhocp_repo() {
+    local -r rhocp=$1
+    local -r major=$2
+    local -r minor=$3
+    local -r arch=$4
+
+    # The repository may be empty if the beta mirror is not up yet
+    if [[ -z "${rhocp}" ]] ; then
+        return
+    fi
+
+    if [[ "${rhocp}" =~ ^[0-9]{1,2}$ ]]; then
+        configure_cdn_repo \
+            "rhocp-${major}.${rhocp}" \
+            "Red Hat OpenShift ${major}.${rhocp} for RHEL 9" \
+            "https://cdn.redhat.com/content/dist/layered/rhel9/${arch}/rhocp/${major}.${rhocp}/os"
+    elif [[ "${rhocp}" =~ ^http ]]; then
+        local -r ocp_repo_name="rhocp-${major}.${minor}-for-rhel-9-mirrorbeta-rpms"
+        local -r tmp_file=$(mktemp)
+
+        tee "${tmp_file}" >/dev/null <<EOF
+[${ocp_repo_name}]
+name=Beta rhocp RPMs for RHEL 9
+baseurl=${rhocp}
+enabled=1
+gpgcheck=0
+skip_if_unavailable=0
+EOF
+        copy_file_to_vm host1 "${tmp_file}" "${tmp_file}"
+        run_command_on_vm host1 "sudo cp ${tmp_file} /etc/yum.repos.d/${ocp_repo_name}.repo"
+        rm -f "${tmp_file}"
+    fi
+}
+
+scenario_create_vms() {
+    prepare_kickstart host1 kickstart-liveimg.ks.template ""
+    launch_vm rhel102-installer
+
+    # Open the firewall ports. Other scenarios get this behavior by
+    # embedding settings in the blueprint, but there is no blueprint
+    # for this scenario. We need do this step before running the RF
+    # suite so that suite can assume it can reach all of the same
+    # ports as for any other test.
+    configure_vm_firewall host1
+
+    # Register the host with subscription manager so we can install
+    # dependencies.
+    subscription_manager_register host1
+}
+
+scenario_remove_vms() {
+    remove_vm host1
+}
+
+scenario_run_tests() {
+    local -r source_reponame=$(basename "${LOCAL_REPO}")
+    local -r source_repo_url="${WEB_SERVER_URL}/rpm-repos/${source_reponame}"
+    local -r target_version=$(local_rpm_version)
+    local -r arch=$(uname -m)
+
+    # Enable the rhocp and dependency repositories.
+    #
+    # Note that rhocp or beta dependencies repository may not yet exist
+    # for the current version. Then, just use whatever we can get for
+    # the previous minor version.
+    configure_rhocp_repo "${RHOCP_MINOR_Y}"       4 "${MINOR_VERSION}"       "${arch}"
+    configure_rhocp_repo "${RHOCP_MINOR_Y_BETA}"  4 "${MINOR_VERSION}"       "${arch}"
+    configure_rhocp_repo "${RHOCP_MINOR_Y1}"      4 "${PREVIOUS_MINOR_VERSION}" "${arch}"
+    configure_rhocp_repo "${RHOCP_MINOR_Y1_BETA}" 4 "${PREVIOUS_MINOR_VERSION}" "${arch}"
+    configure_microshift_mirror "${PREVIOUS_RELEASE_REPO}"
+    configure_cdn_repo \
+        "fast-datapath" \
+        "Red Hat Fast Datapath for RHEL 9" \
+        "https://cdn.redhat.com/content/dist/layered/rhel9/${arch}/fast-datapath/os"
+
+    run_tests host1 \
+        --exitonfailure \
+        --variable "SOURCE_REPO_URL:${source_repo_url}" \
+        --variable "TARGET_VERSION:${target_version}" \
+        --variable "PREVIOUS_MINOR_VERSION:${PREVIOUS_MINOR_VERSION}" \
+        suites/rpm/install.robot \
+        suites/rpm/remove.robot \
+        suites/rpm/upgrade-successful.robot
+}