diff --git a/test/bin/common.sh b/test/bin/common.sh index ac661117ff..8aa5ac464d 100644 --- a/test/bin/common.sh +++ b/test/bin/common.sh @@ -80,7 +80,7 @@ export BOOTC_ISO_DIR="${IMAGEDIR}/bootc-iso-images" # as they are run. # # The CI system will override this, but we need a default for local -# use. Use the image directoy, since that is already served by a web +# use. Use the image directory, since that is already served by a web # server. # # shellcheck disable=SC2034 # used elsewhere @@ -109,7 +109,7 @@ RF_VENV=${RF_VENV:-${OUTPUTDIR}/robotenv} # The location of the gomplate binary. # shellcheck disable=SC2034 # used elsewhere -GOMPLATE=${OUTPUTDIR}/bin/gomplate +export GOMPLATE=${OUTPUTDIR}/bin/gomplate # Which port the web server should run on. WEB_SERVER_PORT=${WEB_SERVER_PORT:-8080} diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index eefac0c3b6..c2bf7ac31f 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -29,7 +29,10 @@ NEXT_REPO = common.get_env_var('NEXT_REPO') HOME_DIR = common.get_env_var("HOME") PULL_SECRET = common.get_env_var('PULL_SECRET', f"{HOME_DIR}/.pull-secret.json") +# Switch to registry.redhat.io/rhel9/bootc-image-builder:9.4 when all the +# required features are supported BIB_IMAGE = "quay.io/centos-bootc/bootc-image-builder:latest" +GOMPLATE = common.get_env_var('GOMPLATE') FORCE_REBUILD = False @@ -121,6 +124,8 @@ def set_rpm_version_info_vars(): previous_version_repo = common.run_command_in_shell(f"source {SCRIPTDIR}/get_rel_version_repo.sh; get_rel_version_repo {PREVIOUS_MINOR_VERSION}") PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO = previous_version_repo.split(',') + RHOCP_MINOR_Y = "" + RHOCP_MINOR_Y1 = "" if is_rhocp_available(MINOR_VERSION): RHOCP_MINOR_Y = MINOR_VERSION if is_rhocp_available(PREVIOUS_MINOR_VERSION): @@ -134,6 +139,21 @@ def set_rpm_version_info_vars(): YMINUS2_RELEASE_REPO = common.run_command_in_shell(f"source {SCRIPTDIR}/get_rel_version_repo.sh; get_ocp_repo_name_for_version {YMINUS2_MINOR_VERSION}") RHOCP_MINOR_Y2 = YMINUS2_MINOR_VERSION + # Update environment variables based on the RPM version global variables. + # These are used for templating container files and images. + rpmver_globals_vars = [ + 'SOURCE_VERSION', 'MINOR_VERSION', 'PREVIOUS_MINOR_VERSION', + 'YMINUS2_MINOR_VERSION', 'FAKE_NEXT_MINOR_VERSION', 'SOURCE_VERSION_BASE', + 'CURRENT_RELEASE_VERSION', 'CURRENT_RELEASE_REPO', 'PREVIOUS_RELEASE_VERSION', + 'PREVIOUS_RELEASE_REPO', 'RHOCP_MINOR_Y', 'RHOCP_MINOR_Y1', + 'RHOCP_MINOR_Y2', 'YMINUS2_RELEASE_VERSION', 'YMINUS2_RELEASE_REPO' + ] + for var in rpmver_globals_vars: + value = globals().get(var) + if value is None: + raise Exception(f"The '{var}' global variable does not exist") + os.environ[var] = str(value) + def get_container_images(path, version): # Find the last microshift-release-info RPM with the specified version @@ -180,6 +200,29 @@ def extract_container_images(version, repo_spec, outfile, dry_run=False): common.popd() +def run_template_cmd(ifile, ofile, dry_run): + # Remove the .template suffix from the output file + ofile = ofile.removesuffix(".template") + # Run the templating command + gomplate_args = [ + GOMPLATE, + "--file", ifile, + "--out", ofile + ] + common.run_command_in_shell(gomplate_args, dry_run) + + +def process_template_dir(idir, odir, dry_run): + # Create the output directory + os.makedirs(odir, exist_ok=True) + # Process the input directory running templating on the files + # and copying them to the output directory + for file in os.listdir(idir): + ifile = os.path.join(idir, file) + ofile = os.path.join(odir, common.basename(ifile)) + run_template_cmd(ifile, ofile, dry_run) + + def process_containerfile(groupdir, containerfile, dry_run): cf_path = os.path.join(groupdir, containerfile) cf_outname = os.path.splitext(containerfile)[0] @@ -204,7 +247,7 @@ def process_containerfile(groupdir, containerfile, dry_run): "sudo", "podman", "build", "--authfile", PULL_SECRET, "-t", cf_outname, "-f", cf_path, - os.path.join(IMAGEDIR, "rpm-repos") + IMAGEDIR ] common.run_command_in_shell(build_args, dry_run, logfile, logfile) common.record_junit(cf_path, "build-container", "OK") @@ -255,9 +298,20 @@ def process_image_bootc(groupdir, bootcfile, dry_run): try: # Redirect the output to the log file with open(bf_logfile, 'w') as logfile: + # Download the bootc image builder itself in case + # it requires authorization for accessing the image + pull_args = [ + "sudo", "podman", "pull", + "--authfile", PULL_SECRET, BIB_IMAGE + ] + common.run_command_in_shell(pull_args, dry_run, logfile, logfile) + common.record_junit(bf_path, "pull-bootc-bib", "OK") + + # Read the image reference bf_imgref = common.read_file(bf_path).strip() + + # If not already local, download the image to be used by bootc image builder if not bf_imgref.startswith('localhost/'): - # If not already local, download the image to be used by bootc image builder pull_args = [ "sudo", "podman", "pull", "--authfile", PULL_SECRET, bf_imgref @@ -371,6 +425,13 @@ def main(): global FORCE_REBUILD if args.force_rebuild: FORCE_REBUILD = True + # Fetch gomplate if necessary + if not os.path.exists(GOMPLATE): + gomplate_args = [ + f"{SCRIPTDIR}/../../scripts/fetch_tools.sh", + "gomplate" + ] + common.run_command(gomplate_args, args.dry_run) # Determine versions of RPM packages set_rpm_version_info_vars() @@ -385,6 +446,11 @@ def main(): extract_container_images(PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO, CONTAINER_LIST, args.dry_run) extract_container_images(YMINUS2_RELEASE_VERSION, YMINUS2_RELEASE_REPO, CONTAINER_LIST, args.dry_run) + # Process template files + process_template_dir( + os.path.join(SCRIPTDIR, "../bootc-sources"), + os.path.join(IMAGEDIR, "bootc-sources"), + args.dry_run) # Process individual group directory if args.group_dir: process_group(args.group_dir, args.build_type, args.dry_run) diff --git a/test/bootc-sources/microshift_repo_config.sh.template b/test/bootc-sources/microshift_repo_config.sh.template new file mode 100644 index 0000000000..34979671aa --- /dev/null +++ b/test/bootc-sources/microshift_repo_config.sh.template @@ -0,0 +1,115 @@ +#!/bin/bash +set -euo pipefail + +USHIFT_LOCAL_REPO_FILE=/etc/yum.repos.d/microshift-local.repo +OCP_MIRROR_REPO_FILE=/etc/yum.repos.d/openshift-mirror-beta.repo +OCP_DTPATH_REPO_FILE=/etc/yum.repos.d/openshift-fast-datapath.repo +OCP_RHOCP_REPO_FILE=/etc/yum.repos.d/openshift-rhocp.repo + +usage() { + echo "Usage: $(basename $0) <-create microshift_local_repo_path | -delete>" + [ -n "$1" ] && echo "ERROR: $1" + exit 1 +} + +del_repo_files() { + rm -f "${USHIFT_LOCAL_REPO_FILE}" + rm -f "${OCP_MIRROR_REPO_FILE}" + rm -f "${OCP_DTPATH_REPO_FILE}" + rm -f "${OCP_RHOCP_REPO_FILE}" +} + +config_local_repos() { + local -r repo_path=$1 + cat > "${USHIFT_LOCAL_REPO_FILE}" < "${OCP_MIRROR_REPO_FILE}" < "${OCP_DTPATH_REPO_FILE}" < "${OCP_RHOCP_REPO_FILE}" < "/etc/yum.repos.d/microshift-local.repo" +# Copy the repository configuration script +COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +# Copy the MicroShift repository contents +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH # Implement workarounds necessary for the successful MicroShift operation # - The /var/run directory must be a symbolic link to /run, which is not @@ -34,12 +18,16 @@ skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.r # is accessed by NetworkManager at /var/run/dbus. RUN [ ! -L /var/run ] && rm -rf /var/run && ln -s /run /var/ +# Add the following repositories and print their contents: +# - MicroShift local RPM repository +# - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies # Install MicroShift, few helper utilities and cleanup -RUN dnf install -y vi firewalld microshift && \ +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y vi firewalld microshift && \ systemctl enable microshift && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all # Configure firewall diff --git a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile index 6433abea45..68b1d22b8b 100644 --- a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile @@ -1,31 +1,15 @@ FROM registry.redhat.io/rhel9/rhel-bootc:9.4 # Build arguments -ARG PREVIOUS_MINOR_VERSION=15 +ARG REPO_CONFIG_SCRIPT=/tmp/microshift_repo_config.sh ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME -# Copy the MicroShift repository contents -COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH - -# MicroShift local RPM repository -RUN printf "\ -[microshift-local]\n\ -name=MicroShift Local Repository\n\ -baseurl=%s\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" +# Copy the repository configuration script +COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +# Copy the MicroShift repository contents +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH # Implement workarounds necessary for the successful MicroShift operation # - The /var/run directory must be a symbolic link to /run, which is not @@ -34,12 +18,17 @@ skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.r # is accessed by NetworkManager at /var/run/dbus. RUN [ ! -L /var/run ] && rm -rf /var/run && ln -s /run /var/ +# Add the following repositories and print their contents: +# - MicroShift local RPM repository +# - OpenShift previous minor version RHEL repositories for MicroShift dependencies # Install MicroShift, few helper utilities and cleanup -RUN dnf install -y vi firewalld microshift && \ +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + awk 'FNR==1 {print "=== " FILENAME " ==="} {print}' /etc/yum.repos.d/*.repo && \ + dnf install -y vi firewalld microshift && \ systemctl enable microshift && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all # Configure firewall diff --git a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile index 785ba7210b..c4ac8f94c8 100644 --- a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile @@ -1,35 +1,23 @@ FROM localhost/cos9-bootc-source:latest # Build arguments -ARG PREVIOUS_MINOR_VERSION=15 +ARG REPO_CONFIG_SCRIPT=/tmp/microshift_repo_config.sh ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME -# Copy the MicroShift repository contents -COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH - -# MicroShift local RPM repository -RUN printf "\ -[microshift-local]\n\ -name=MicroShift Local Repository\n\ -baseurl=%s\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" +# Copy the repository configuration script +COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +# Copy the MicroShift repository contents +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# Install MicroShift optional RPMs -RUN dnf install -y microshift-olm microshift-multus && \ +# Add the following repositories and print their contents: +# - MicroShift local RPM repository +# - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies +# Install MicroShift optional packages and cleanup +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y microshift-olm microshift-multus && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all diff --git a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile index 05f87ed46c..561a49f6b9 100644 --- a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile @@ -1,35 +1,23 @@ FROM localhost/rhel94-bootc-source:latest # Build arguments -ARG PREVIOUS_MINOR_VERSION=15 +ARG REPO_CONFIG_SCRIPT=/tmp/microshift_repo_config.sh ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME -# Copy the MicroShift repository contents -COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH - -# MicroShift local RPM repository -RUN printf "\ -[microshift-local]\n\ -name=MicroShift Local Repository\n\ -baseurl=%s\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" +# Copy the repository configuration script +COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +# Copy the MicroShift repository contents +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# Install MicroShift optional RPMs -RUN dnf install -y microshift-olm microshift-multus && \ +# Add the following repositories and print their contents: +# - MicroShift local RPM repository +# - OpenShift previous minor version RHEL repositories for MicroShift dependencies +# Install MicroShift optional packages and cleanup +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y microshift-olm microshift-multus && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all