From 381d0c34acd5d4f81324bd9cee2ee2f9769b358d Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 4 May 2024 09:07:26 +0000 Subject: [PATCH 1/7] Add support for gomplate in bootc image and container files --- test/bin/common.sh | 4 +- test/bin/pyutils/build_bootc_images.py | 62 +++++++++++++++++-- .../group1/cos9-bootc-source.containerfile | 5 +- .../group1/rhel94-bootc-source.containerfile | 5 +- .../cos9-bootc-source-optionals.containerfile | 5 +- ...hel94-bootc-source-optionals.containerfile | 5 +- 6 files changed, 68 insertions(+), 18 deletions(-) diff --git a/test/bin/common.sh b/test/bin/common.sh index ac661117ff..8aa5ac464d 100644 --- a/test/bin/common.sh +++ b/test/bin/common.sh @@ -80,7 +80,7 @@ export BOOTC_ISO_DIR="${IMAGEDIR}/bootc-iso-images" # as they are run. # # The CI system will override this, but we need a default for local -# use. Use the image directoy, since that is already served by a web +# use. Use the image directory, since that is already served by a web # server. # # shellcheck disable=SC2034 # used elsewhere @@ -109,7 +109,7 @@ RF_VENV=${RF_VENV:-${OUTPUTDIR}/robotenv} # The location of the gomplate binary. # shellcheck disable=SC2034 # used elsewhere -GOMPLATE=${OUTPUTDIR}/bin/gomplate +export GOMPLATE=${OUTPUTDIR}/bin/gomplate # Which port the web server should run on. WEB_SERVER_PORT=${WEB_SERVER_PORT:-8080} diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index eefac0c3b6..c54abc82cf 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -29,7 +29,10 @@ NEXT_REPO = common.get_env_var('NEXT_REPO') HOME_DIR = common.get_env_var("HOME") PULL_SECRET = common.get_env_var('PULL_SECRET', f"{HOME_DIR}/.pull-secret.json") +# Switch to registry.redhat.io/rhel9/bootc-image-builder:9.4 when all the +# required features are supported BIB_IMAGE = "quay.io/centos-bootc/bootc-image-builder:latest" +GOMPLATE = common.get_env_var('GOMPLATE') FORCE_REBUILD = False @@ -121,6 +124,8 @@ def set_rpm_version_info_vars(): previous_version_repo = common.run_command_in_shell(f"source {SCRIPTDIR}/get_rel_version_repo.sh; get_rel_version_repo {PREVIOUS_MINOR_VERSION}") PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO = previous_version_repo.split(',') + RHOCP_MINOR_Y = "" + RHOCP_MINOR_Y1 = "" if is_rhocp_available(MINOR_VERSION): RHOCP_MINOR_Y = MINOR_VERSION if is_rhocp_available(PREVIOUS_MINOR_VERSION): @@ -134,6 +139,21 @@ def set_rpm_version_info_vars(): YMINUS2_RELEASE_REPO = common.run_command_in_shell(f"source {SCRIPTDIR}/get_rel_version_repo.sh; get_ocp_repo_name_for_version {YMINUS2_MINOR_VERSION}") RHOCP_MINOR_Y2 = YMINUS2_MINOR_VERSION + # Update environment variables based on the RPM version global variables. + # These are used for templating container files and images. + rpmver_globals_vars = [ + 'SOURCE_VERSION', 'MINOR_VERSION', 'PREVIOUS_MINOR_VERSION', + 'YMINUS2_MINOR_VERSION', 'FAKE_NEXT_MINOR_VERSION', 'SOURCE_VERSION_BASE', + 'CURRENT_RELEASE_VERSION', 'CURRENT_RELEASE_REPO', 'PREVIOUS_RELEASE_VERSION', + 'PREVIOUS_RELEASE_REPO', 'RHOCP_MINOR_Y', 'RHOCP_MINOR_Y1', + 'RHOCP_MINOR_Y2', 'YMINUS2_RELEASE_VERSION', 'YMINUS2_RELEASE_REPO' + ] + for var in rpmver_globals_vars: + value = globals().get(var) + if value is None: + raise Exception(f"The '{var}' global variable does not exist") + os.environ[var] = str(value) + def get_container_images(path, version): # Find the last microshift-release-info RPM with the specified version @@ -180,10 +200,20 @@ def extract_container_images(version, repo_spec, outfile, dry_run=False): common.popd() +def run_gomplate(ifile, ofile, dry_run, logfile): + gomplate_args = [ + GOMPLATE, + "--file", ifile, + "--out", ofile + ] + common.run_command_in_shell(gomplate_args, dry_run, logfile, logfile) + + def process_containerfile(groupdir, containerfile, dry_run): cf_path = os.path.join(groupdir, containerfile) cf_outname = os.path.splitext(containerfile)[0] cf_outdir = os.path.join(BOOTC_IMAGE_DIR, cf_outname) + cf_gomplate = os.path.join(BOOTC_IMAGE_DIR, f"{containerfile}.gomplate") cf_logfile = os.path.join(BOOTC_IMAGE_DIR, f"{cf_outname}.log") cf_targetimg = os.path.join(cf_outdir, "index.json") @@ -199,11 +229,14 @@ def process_containerfile(groupdir, containerfile, dry_run): try: # Redirect the output to the log file with open(cf_logfile, 'w') as logfile: - # Run the container build command + # Run gomplate on the input container file + run_gomplate(cf_path, cf_gomplate, dry_run, logfile) + + # Run the container build command, using the templated file as an input build_args = [ "sudo", "podman", "build", "--authfile", PULL_SECRET, - "-t", cf_outname, "-f", cf_path, + "-t", cf_outname, "-f", cf_gomplate, os.path.join(IMAGEDIR, "rpm-repos") ] common.run_command_in_shell(build_args, dry_run, logfile, logfile) @@ -239,6 +272,7 @@ def process_image_bootc(groupdir, bootcfile, dry_run): bf_path = os.path.join(groupdir, bootcfile) bf_outname = os.path.splitext(bootcfile)[0] bf_outdir = os.path.join(BOOTC_ISO_DIR, bf_outname) + bf_gomplate = os.path.join(BOOTC_IMAGE_DIR, f"{bootcfile}.gomplate") bf_logfile = os.path.join(BOOTC_ISO_DIR, f"{bf_outname}.log") bf_targetiso = os.path.join(VM_DISK_BASEDIR, f"{bf_outname}.iso") @@ -255,9 +289,22 @@ def process_image_bootc(groupdir, bootcfile, dry_run): try: # Redirect the output to the log file with open(bf_logfile, 'w') as logfile: - bf_imgref = common.read_file(bf_path).strip() + # Download the bootc image builder itself in case + # it requires authorization for accessing the image + pull_args = [ + "sudo", "podman", "pull", + "--authfile", PULL_SECRET, BIB_IMAGE + ] + common.run_command_in_shell(pull_args, dry_run, logfile, logfile) + common.record_junit(bf_path, "pull-bootc-bib", "OK") + + # Run gomplate on the input image file + run_gomplate(bf_path, bf_gomplate, dry_run, logfile) + # Read the image reference using the templated file as an input + bf_imgref = common.read_file(bf_gomplate).strip() + + # If not already local, download the image to be used by bootc image builder if not bf_imgref.startswith('localhost/'): - # If not already local, download the image to be used by bootc image builder pull_args = [ "sudo", "podman", "pull", "--authfile", PULL_SECRET, bf_imgref @@ -371,6 +418,13 @@ def main(): global FORCE_REBUILD if args.force_rebuild: FORCE_REBUILD = True + # Fetch gomplate if necessary + if not os.path.exists(GOMPLATE): + gomplate_args = [ + f"{SCRIPTDIR}/../../scripts/fetch_tools.sh", + "gomplate" + ] + common.run_command(gomplate_args, args.dry_run) # Determine versions of RPM packages set_rpm_version_info_vars() diff --git a/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile index 763180f990..7d24d8f478 100644 --- a/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile @@ -1,7 +1,6 @@ FROM quay.io/centos-bootc/centos-bootc:stream9 # Build arguments -ARG PREVIOUS_MINOR_VERSION=15 ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME @@ -22,10 +21,10 @@ skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshif RUN printf "\ [openshift-mirror-beta]\n\ name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ +baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" # Implement workarounds necessary for the successful MicroShift operation # - The /var/run directory must be a symbolic link to /run, which is not diff --git a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile index 6433abea45..c962ebf4cf 100644 --- a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile @@ -1,7 +1,6 @@ FROM registry.redhat.io/rhel9/rhel-bootc:9.4 # Build arguments -ARG PREVIOUS_MINOR_VERSION=15 ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME @@ -22,10 +21,10 @@ skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshif RUN printf "\ [openshift-mirror-beta]\n\ name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ +baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" # Implement workarounds necessary for the successful MicroShift operation # - The /var/run directory must be a symbolic link to /run, which is not diff --git a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile index 785ba7210b..2dd9d0bdbb 100644 --- a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile @@ -1,7 +1,6 @@ FROM localhost/cos9-bootc-source:latest # Build arguments -ARG PREVIOUS_MINOR_VERSION=15 ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME @@ -22,10 +21,10 @@ skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshif RUN printf "\ [openshift-mirror-beta]\n\ name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ +baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" # Install MicroShift optional RPMs RUN dnf install -y microshift-olm microshift-multus && \ diff --git a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile index 05f87ed46c..bfb88679bf 100644 --- a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile @@ -1,7 +1,6 @@ FROM localhost/rhel94-bootc-source:latest # Build arguments -ARG PREVIOUS_MINOR_VERSION=15 ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME @@ -22,10 +21,10 @@ skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshif RUN printf "\ [openshift-mirror-beta]\n\ name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/%s/dependencies/rpms/4.%s-el9-beta/\n\ +baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "$(uname -m)" "${PREVIOUS_MINOR_VERSION}" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" # Install MicroShift optional RPMs RUN dnf install -y microshift-olm microshift-multus && \ From 6ada90bf7c2d0fb07b40fc5f7a67306784bbb1a1 Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Sat, 4 May 2024 09:38:25 +0000 Subject: [PATCH 2/7] Switch to using rhocp and fastdatapath repos in rhel images --- .../group1/cos9-bootc-source.containerfile | 11 +++-- .../group1/rhel94-bootc-source.containerfile | 43 +++++++++++++------ .../cos9-bootc-source-optionals.containerfile | 11 +++-- ...hel94-bootc-source-optionals.containerfile | 43 +++++++++++++------ 4 files changed, 72 insertions(+), 36 deletions(-) diff --git a/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile index 7d24d8f478..d2b780c7e3 100644 --- a/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile @@ -7,18 +7,17 @@ ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME # Copy the MicroShift repository contents COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# MicroShift local RPM repository +# Add the following repositories: +# - MicroShift local RPM repository +# - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies RUN printf "\ [microshift-local]\n\ name=MicroShift Local Repository\n\ baseurl=%s\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" - -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ +skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" && \ + printf "\ [openshift-mirror-beta]\n\ name=OpenShift Mirror Beta Repository\n\ baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ diff --git a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile index c962ebf4cf..7cc004eaed 100644 --- a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile @@ -7,24 +7,43 @@ ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME # Copy the MicroShift repository contents COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# MicroShift local RPM repository -RUN printf "\ +# Add the following repositories, using host entitlements for SSL keys: +# - MicroShift local RPM repository +# - Fast Datapath repository for MicroShift dependencies +# - OpenShift previous minor version repository for MicroShift dependencies +RUN sslkey=$(find /etc/pki/entitlement-host/ -type f -name "*-key.pem" -print -quit) && \ + sslcrt=$(find /etc/pki/entitlement-host/ -type f -name "*.pem" ! -name "*-key.pem" -print -quit) && \ + printf "\ [microshift-local]\n\ name=MicroShift Local Repository\n\ baseurl=%s\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" - -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ +skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" && \ + printf "\ +[openshift-fast-datapath]\n\ +name=Fast Datapath for RHEL 9\n\ +baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/fast-datapath/os\n\ enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +gpgcheck=1\n\ +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ +sslverify=1\n\ +sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ +sslclientkey = %s\n\ +sslclientcert = %s\n\ +skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-fast-datapath.repo" && \ + printf "\ +[openshift-rhocp]\n\ +name=OpenShift Dependencies RHEL 9\n\ +baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/rhocp/4.{{ .Env.PREVIOUS_MINOR_VERSION }}/os\n\ +enabled=1\n\ +gpgcheck=1\n\ +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ +sslverify=1\n\ +sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ +sslclientkey = %s\n\ +sslclientcert = %s\n\ +skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-rhocp.repo" # Implement workarounds necessary for the successful MicroShift operation # - The /var/run directory must be a symbolic link to /run, which is not diff --git a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile index 2dd9d0bdbb..460f1da271 100644 --- a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile @@ -7,18 +7,17 @@ ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME # Copy the MicroShift repository contents COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# MicroShift local RPM repository +# Add the following repositories: +# - MicroShift local RPM repository +# - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies RUN printf "\ [microshift-local]\n\ name=MicroShift Local Repository\n\ baseurl=%s\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" - -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ +skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" && \ + printf "\ [openshift-mirror-beta]\n\ name=OpenShift Mirror Beta Repository\n\ baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ diff --git a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile index bfb88679bf..ad5b8a3be3 100644 --- a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile @@ -7,24 +7,43 @@ ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME # Copy the MicroShift repository contents COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# MicroShift local RPM repository -RUN printf "\ +# Add the following repositories, using host entitlements for SSL keys: +# - MicroShift local RPM repository +# - Fast Datapath repository for MicroShift dependencies +# - OpenShift previous minor version repository for MicroShift dependencies +RUN sslkey=$(find /etc/pki/entitlement-host/ -type f -name "*-key.pem" -print -quit) && \ + sslcrt=$(find /etc/pki/entitlement-host/ -type f -name "*.pem" ! -name "*-key.pem" -print -quit) && \ + printf "\ [microshift-local]\n\ name=MicroShift Local Repository\n\ baseurl=%s\n\ enabled=1\n\ gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" - -# OpenShift Mirror beta RPM repository for MicroShift dependencies -# Only the released previous minor version is guaranteed to be available -RUN printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ +skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" && \ + printf "\ +[openshift-fast-datapath]\n\ +name=Fast Datapath for RHEL 9\n\ +baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/fast-datapath/os\n\ enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +gpgcheck=1\n\ +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ +sslverify=1\n\ +sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ +sslclientkey = %s\n\ +sslclientcert = %s\n\ +skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-fast-datapath.repo" && \ + printf "\ +[openshift-rhocp]\n\ +name=OpenShift Dependencies RHEL 9\n\ +baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/rhocp/4.{{ .Env.PREVIOUS_MINOR_VERSION }}/os\n\ +enabled=1\n\ +gpgcheck=1\n\ +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ +sslverify=1\n\ +sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ +sslclientkey = %s\n\ +sslclientcert = %s\n\ +skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-rhocp.repo" # Install MicroShift optional RPMs RUN dnf install -y microshift-olm microshift-multus && \ From 3c4f0ec7c70d9f51715b14c22f8a6b4516a0002f Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Wed, 8 May 2024 05:34:37 +0000 Subject: [PATCH 3/7] Fix template destination file names --- test/bin/pyutils/build_bootc_images.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index c54abc82cf..b64a8abb5a 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -213,7 +213,7 @@ def process_containerfile(groupdir, containerfile, dry_run): cf_path = os.path.join(groupdir, containerfile) cf_outname = os.path.splitext(containerfile)[0] cf_outdir = os.path.join(BOOTC_IMAGE_DIR, cf_outname) - cf_gomplate = os.path.join(BOOTC_IMAGE_DIR, f"{containerfile}.gomplate") + cf_processed = os.path.join(BOOTC_IMAGE_DIR, containerfile) cf_logfile = os.path.join(BOOTC_IMAGE_DIR, f"{cf_outname}.log") cf_targetimg = os.path.join(cf_outdir, "index.json") @@ -230,13 +230,13 @@ def process_containerfile(groupdir, containerfile, dry_run): # Redirect the output to the log file with open(cf_logfile, 'w') as logfile: # Run gomplate on the input container file - run_gomplate(cf_path, cf_gomplate, dry_run, logfile) + run_gomplate(cf_path, cf_processed, dry_run, logfile) # Run the container build command, using the templated file as an input build_args = [ "sudo", "podman", "build", "--authfile", PULL_SECRET, - "-t", cf_outname, "-f", cf_gomplate, + "-t", cf_outname, "-f", cf_processed, os.path.join(IMAGEDIR, "rpm-repos") ] common.run_command_in_shell(build_args, dry_run, logfile, logfile) @@ -272,7 +272,7 @@ def process_image_bootc(groupdir, bootcfile, dry_run): bf_path = os.path.join(groupdir, bootcfile) bf_outname = os.path.splitext(bootcfile)[0] bf_outdir = os.path.join(BOOTC_ISO_DIR, bf_outname) - bf_gomplate = os.path.join(BOOTC_IMAGE_DIR, f"{bootcfile}.gomplate") + bf_processed = os.path.join(BOOTC_ISO_DIR, bootcfile) bf_logfile = os.path.join(BOOTC_ISO_DIR, f"{bf_outname}.log") bf_targetiso = os.path.join(VM_DISK_BASEDIR, f"{bf_outname}.iso") @@ -299,9 +299,9 @@ def process_image_bootc(groupdir, bootcfile, dry_run): common.record_junit(bf_path, "pull-bootc-bib", "OK") # Run gomplate on the input image file - run_gomplate(bf_path, bf_gomplate, dry_run, logfile) + run_gomplate(bf_path, bf_processed, dry_run, logfile) # Read the image reference using the templated file as an input - bf_imgref = common.read_file(bf_gomplate).strip() + bf_imgref = common.read_file(bf_processed).strip() # If not already local, download the image to be used by bootc image builder if not bf_imgref.startswith('localhost/'): From a53d3c9885f7d55b9d0bd2370dc964140f4d111a Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Wed, 8 May 2024 15:57:59 +0000 Subject: [PATCH 4/7] Reimplement templating to work on bootc-sources contents --- test/bin/pyutils/build_bootc_images.py | 40 ++++--- .../microshift_repo_config.sh.template | 112 ++++++++++++++++++ .../group1/cos9-bootc-source.containerfile | 34 ++---- .../group1/rhel94-bootc-source.containerfile | 54 ++------- .../cos9-bootc-source-optionals.containerfile | 30 ++--- ...hel94-bootc-source-optionals.containerfile | 54 ++------- 6 files changed, 184 insertions(+), 140 deletions(-) create mode 100644 test/bootc-sources/microshift_repo_config.sh.template diff --git a/test/bin/pyutils/build_bootc_images.py b/test/bin/pyutils/build_bootc_images.py index b64a8abb5a..c2bf7ac31f 100644 --- a/test/bin/pyutils/build_bootc_images.py +++ b/test/bin/pyutils/build_bootc_images.py @@ -200,20 +200,33 @@ def extract_container_images(version, repo_spec, outfile, dry_run=False): common.popd() -def run_gomplate(ifile, ofile, dry_run, logfile): +def run_template_cmd(ifile, ofile, dry_run): + # Remove the .template suffix from the output file + ofile = ofile.removesuffix(".template") + # Run the templating command gomplate_args = [ GOMPLATE, "--file", ifile, "--out", ofile ] - common.run_command_in_shell(gomplate_args, dry_run, logfile, logfile) + common.run_command_in_shell(gomplate_args, dry_run) + + +def process_template_dir(idir, odir, dry_run): + # Create the output directory + os.makedirs(odir, exist_ok=True) + # Process the input directory running templating on the files + # and copying them to the output directory + for file in os.listdir(idir): + ifile = os.path.join(idir, file) + ofile = os.path.join(odir, common.basename(ifile)) + run_template_cmd(ifile, ofile, dry_run) def process_containerfile(groupdir, containerfile, dry_run): cf_path = os.path.join(groupdir, containerfile) cf_outname = os.path.splitext(containerfile)[0] cf_outdir = os.path.join(BOOTC_IMAGE_DIR, cf_outname) - cf_processed = os.path.join(BOOTC_IMAGE_DIR, containerfile) cf_logfile = os.path.join(BOOTC_IMAGE_DIR, f"{cf_outname}.log") cf_targetimg = os.path.join(cf_outdir, "index.json") @@ -229,15 +242,12 @@ def process_containerfile(groupdir, containerfile, dry_run): try: # Redirect the output to the log file with open(cf_logfile, 'w') as logfile: - # Run gomplate on the input container file - run_gomplate(cf_path, cf_processed, dry_run, logfile) - - # Run the container build command, using the templated file as an input + # Run the container build command build_args = [ "sudo", "podman", "build", "--authfile", PULL_SECRET, - "-t", cf_outname, "-f", cf_processed, - os.path.join(IMAGEDIR, "rpm-repos") + "-t", cf_outname, "-f", cf_path, + IMAGEDIR ] common.run_command_in_shell(build_args, dry_run, logfile, logfile) common.record_junit(cf_path, "build-container", "OK") @@ -272,7 +282,6 @@ def process_image_bootc(groupdir, bootcfile, dry_run): bf_path = os.path.join(groupdir, bootcfile) bf_outname = os.path.splitext(bootcfile)[0] bf_outdir = os.path.join(BOOTC_ISO_DIR, bf_outname) - bf_processed = os.path.join(BOOTC_ISO_DIR, bootcfile) bf_logfile = os.path.join(BOOTC_ISO_DIR, f"{bf_outname}.log") bf_targetiso = os.path.join(VM_DISK_BASEDIR, f"{bf_outname}.iso") @@ -298,10 +307,8 @@ def process_image_bootc(groupdir, bootcfile, dry_run): common.run_command_in_shell(pull_args, dry_run, logfile, logfile) common.record_junit(bf_path, "pull-bootc-bib", "OK") - # Run gomplate on the input image file - run_gomplate(bf_path, bf_processed, dry_run, logfile) - # Read the image reference using the templated file as an input - bf_imgref = common.read_file(bf_processed).strip() + # Read the image reference + bf_imgref = common.read_file(bf_path).strip() # If not already local, download the image to be used by bootc image builder if not bf_imgref.startswith('localhost/'): @@ -439,6 +446,11 @@ def main(): extract_container_images(PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO, CONTAINER_LIST, args.dry_run) extract_container_images(YMINUS2_RELEASE_VERSION, YMINUS2_RELEASE_REPO, CONTAINER_LIST, args.dry_run) + # Process template files + process_template_dir( + os.path.join(SCRIPTDIR, "../bootc-sources"), + os.path.join(IMAGEDIR, "bootc-sources"), + args.dry_run) # Process individual group directory if args.group_dir: process_group(args.group_dir, args.build_type, args.dry_run) diff --git a/test/bootc-sources/microshift_repo_config.sh.template b/test/bootc-sources/microshift_repo_config.sh.template new file mode 100644 index 0000000000..478d6b7d53 --- /dev/null +++ b/test/bootc-sources/microshift_repo_config.sh.template @@ -0,0 +1,112 @@ +#!/bin/bash +set -euo pipefail + +USHIFT_LOCAL_REPO_FILE=/etc/yum.repos.d/microshift-local.repo +OCP_MIRROR_REPO_FILE=/etc/yum.repos.d/openshift-mirror-beta.repo +OCP_DTPATH_REPO_FILE=/etc/yum.repos.d/openshift-fast-datapath.repo +OCP_RHOCP_REPO_FILE=/etc/yum.repos.d/openshift-rhocp.repo + +usage() { + echo "Usage: $(basename $0) <-create microshift_local_repo_path | -delete>" + [ -n "$1" ] && echo "ERROR: $1" + exit 1 +} + +del_repo_files() { + rm -f "${USHIFT_LOCAL_REPO_FILE}" + rm -f "${OCP_MIRROR_REPO_FILE}" + rm -f "${OCP_DTPATH_REPO_FILE}" + rm -f "${OCP_RHOCP_REPO_FILE}" +} + +config_local_repos() { + local -r repo_path=$1 + cat > "${USHIFT_LOCAL_REPO_FILE}" < "${OCP_MIRROR_REPO_FILE}" < "${OCP_DTPATH_REPO_FILE}" < "${OCP_RHOCP_REPO_FILE}" < "/etc/yum.repos.d/microshift-local.repo" && \ - printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" +# Copy the MicroShift repository contents +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH # Implement workarounds necessary for the successful MicroShift operation # - The /var/run directory must be a symbolic link to /run, which is not @@ -32,12 +18,16 @@ skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" # is accessed by NetworkManager at /var/run/dbus. RUN [ ! -L /var/run ] && rm -rf /var/run && ln -s /run /var/ +# Add the following repositories: +# - MicroShift local RPM repository +# - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies # Install MicroShift, few helper utilities and cleanup -RUN dnf install -y vi firewalld microshift && \ +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y vi firewalld microshift && \ systemctl enable microshift && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all # Configure firewall diff --git a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile index 7cc004eaed..205b473ada 100644 --- a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile @@ -1,49 +1,15 @@ FROM registry.redhat.io/rhel9/rhel-bootc:9.4 # Build arguments +ARG REPO_CONFIG_SCRIPT=/tmp/microshift_repo_config.sh ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME -# Copy the MicroShift repository contents -COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH +# Copy the repository configuration script +COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} -# Add the following repositories, using host entitlements for SSL keys: -# - MicroShift local RPM repository -# - Fast Datapath repository for MicroShift dependencies -# - OpenShift previous minor version repository for MicroShift dependencies -RUN sslkey=$(find /etc/pki/entitlement-host/ -type f -name "*-key.pem" -print -quit) && \ - sslcrt=$(find /etc/pki/entitlement-host/ -type f -name "*.pem" ! -name "*-key.pem" -print -quit) && \ - printf "\ -[microshift-local]\n\ -name=MicroShift Local Repository\n\ -baseurl=%s\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" && \ - printf "\ -[openshift-fast-datapath]\n\ -name=Fast Datapath for RHEL 9\n\ -baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/fast-datapath/os\n\ -enabled=1\n\ -gpgcheck=1\n\ -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ -sslverify=1\n\ -sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ -sslclientkey = %s\n\ -sslclientcert = %s\n\ -skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-fast-datapath.repo" && \ - printf "\ -[openshift-rhocp]\n\ -name=OpenShift Dependencies RHEL 9\n\ -baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/rhocp/4.{{ .Env.PREVIOUS_MINOR_VERSION }}/os\n\ -enabled=1\n\ -gpgcheck=1\n\ -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ -sslverify=1\n\ -sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ -sslclientkey = %s\n\ -sslclientcert = %s\n\ -skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-rhocp.repo" +# Copy the MicroShift repository contents +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH # Implement workarounds necessary for the successful MicroShift operation # - The /var/run directory must be a symbolic link to /run, which is not @@ -52,12 +18,16 @@ skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-r # is accessed by NetworkManager at /var/run/dbus. RUN [ ! -L /var/run ] && rm -rf /var/run && ln -s /run /var/ +# Add the following repositories: +# - MicroShift local RPM repository +# - OpenShift previous minor version RHEL repositories for MicroShift dependencies # Install MicroShift, few helper utilities and cleanup -RUN dnf install -y vi firewalld microshift && \ +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y vi firewalld microshift && \ systemctl enable microshift && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all # Configure firewall diff --git a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile index 460f1da271..a41f609068 100644 --- a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile @@ -1,33 +1,23 @@ FROM localhost/cos9-bootc-source:latest # Build arguments +ARG REPO_CONFIG_SCRIPT=/tmp/microshift_repo_config.sh ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME +# Copy the repository configuration script +COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} + # Copy the MicroShift repository contents -COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH # Add the following repositories: # - MicroShift local RPM repository # - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies -RUN printf "\ -[microshift-local]\n\ -name=MicroShift Local Repository\n\ -baseurl=%s\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" && \ - printf "\ -[openshift-mirror-beta]\n\ -name=OpenShift Mirror Beta Repository\n\ -baseurl=https://mirror.openshift.com/pub/openshift-v4/{{ .Env.UNAME_M }}/dependencies/rpms/4.{{ .Env.PREVIOUS_MINOR_VERSION }}-el9-beta/\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" > "/etc/yum.repos.d/openshift-mirror-beta.repo" - -# Install MicroShift optional RPMs -RUN dnf install -y microshift-olm microshift-multus && \ +# Install MicroShift optional packages and cleanup +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y microshift-olm microshift-multus && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all diff --git a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile index ad5b8a3be3..c05f4a0b40 100644 --- a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile @@ -1,53 +1,23 @@ FROM localhost/rhel94-bootc-source:latest # Build arguments +ARG REPO_CONFIG_SCRIPT=/tmp/microshift_repo_config.sh ARG USHIFT_RPM_REPO_NAME=microshift-local ARG USHIFT_RPM_REPO_PATH=/tmp/$USHIFT_RPM_REPO_NAME +# Copy the repository configuration script +COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} + # Copy the MicroShift repository contents -COPY $USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH +COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# Add the following repositories, using host entitlements for SSL keys: +# Add the following repositories: # - MicroShift local RPM repository -# - Fast Datapath repository for MicroShift dependencies -# - OpenShift previous minor version repository for MicroShift dependencies -RUN sslkey=$(find /etc/pki/entitlement-host/ -type f -name "*-key.pem" -print -quit) && \ - sslcrt=$(find /etc/pki/entitlement-host/ -type f -name "*.pem" ! -name "*-key.pem" -print -quit) && \ - printf "\ -[microshift-local]\n\ -name=MicroShift Local Repository\n\ -baseurl=%s\n\ -enabled=1\n\ -gpgcheck=0\n\ -skip_if_unavailable=0\n" "${USHIFT_RPM_REPO_PATH}" > "/etc/yum.repos.d/microshift-local.repo" && \ - printf "\ -[openshift-fast-datapath]\n\ -name=Fast Datapath for RHEL 9\n\ -baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/fast-datapath/os\n\ -enabled=1\n\ -gpgcheck=1\n\ -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ -sslverify=1\n\ -sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ -sslclientkey = %s\n\ -sslclientcert = %s\n\ -skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-fast-datapath.repo" && \ - printf "\ -[openshift-rhocp]\n\ -name=OpenShift Dependencies RHEL 9\n\ -baseurl=https://cdn.redhat.com/content/dist/layered/rhel9/{{ .Env.UNAME_M }}/rhocp/4.{{ .Env.PREVIOUS_MINOR_VERSION }}/os\n\ -enabled=1\n\ -gpgcheck=1\n\ -gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release\n\ -sslverify=1\n\ -sslcacert = /etc/rhsm/ca/redhat-uep.pem\n\ -sslclientkey = %s\n\ -sslclientcert = %s\n\ -skip_if_unavailable=0\n" "${sslkey}" "${sslcrt}" > "/etc/yum.repos.d/openshift-rhocp.repo" - -# Install MicroShift optional RPMs -RUN dnf install -y microshift-olm microshift-multus && \ +# - OpenShift previous minor version RHEL repositories for MicroShift dependencies +# Install MicroShift optional packages and cleanup +RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y microshift-olm microshift-multus && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ rm -rf $USHIFT_RPM_REPO_PATH && \ - rm -f /etc/yum.repos.d/microshift*.repo && \ - rm -f /etc/yum.repos.d/openshift*.repo && \ dnf clean all From dfa3bcfb4a4b924d62341953a17135de663863bc Mon Sep 17 00:00:00 2001 From: Gregory Giguashvili Date: Thu, 9 May 2024 10:47:18 +0000 Subject: [PATCH 5/7] Use repository template files instead of configuration script --- .../microshift-local.repo.template | 6 + .../microshift_repo_config.sh.template | 112 ------------------ .../openshift-fast-datapath.repo.template | 11 ++ .../openshift-mirror-beta.repo.template | 6 + .../openshift-rhocp.repo.template | 11 ++ .../group1/cos9-bootc-source.containerfile | 21 ++-- .../group1/rhel94-bootc-source.containerfile | 27 +++-- .../cos9-bootc-source-optionals.containerfile | 21 ++-- ...hel94-bootc-source-optionals.containerfile | 27 +++-- 9 files changed, 90 insertions(+), 152 deletions(-) create mode 100644 test/bootc-sources/microshift-local.repo.template delete mode 100644 test/bootc-sources/microshift_repo_config.sh.template create mode 100644 test/bootc-sources/openshift-fast-datapath.repo.template create mode 100644 test/bootc-sources/openshift-mirror-beta.repo.template create mode 100644 test/bootc-sources/openshift-rhocp.repo.template diff --git a/test/bootc-sources/microshift-local.repo.template b/test/bootc-sources/microshift-local.repo.template new file mode 100644 index 0000000000..9025341efe --- /dev/null +++ b/test/bootc-sources/microshift-local.repo.template @@ -0,0 +1,6 @@ +[microshift-local] +name=MicroShift Local Repository +baseurl=REPLACE_USHIFT_RPM_REPO_PATH +enabled=1 +gpgcheck=0 +skip_if_unavailable=0 diff --git a/test/bootc-sources/microshift_repo_config.sh.template b/test/bootc-sources/microshift_repo_config.sh.template deleted file mode 100644 index 478d6b7d53..0000000000 --- a/test/bootc-sources/microshift_repo_config.sh.template +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -set -euo pipefail - -USHIFT_LOCAL_REPO_FILE=/etc/yum.repos.d/microshift-local.repo -OCP_MIRROR_REPO_FILE=/etc/yum.repos.d/openshift-mirror-beta.repo -OCP_DTPATH_REPO_FILE=/etc/yum.repos.d/openshift-fast-datapath.repo -OCP_RHOCP_REPO_FILE=/etc/yum.repos.d/openshift-rhocp.repo - -usage() { - echo "Usage: $(basename $0) <-create microshift_local_repo_path | -delete>" - [ -n "$1" ] && echo "ERROR: $1" - exit 1 -} - -del_repo_files() { - rm -f "${USHIFT_LOCAL_REPO_FILE}" - rm -f "${OCP_MIRROR_REPO_FILE}" - rm -f "${OCP_DTPATH_REPO_FILE}" - rm -f "${OCP_RHOCP_REPO_FILE}" -} - -config_local_repos() { - local -r repo_path=$1 - cat > "${USHIFT_LOCAL_REPO_FILE}" < "${OCP_MIRROR_REPO_FILE}" < "${OCP_DTPATH_REPO_FILE}" < "${OCP_RHOCP_REPO_FILE}" < Date: Thu, 9 May 2024 12:02:11 +0000 Subject: [PATCH 6/7] Revert "Use repository template files instead of configuration script" This reverts commit dfa3bcfb4a4b924d62341953a17135de663863bc. --- .../microshift-local.repo.template | 6 - .../microshift_repo_config.sh.template | 112 ++++++++++++++++++ .../openshift-fast-datapath.repo.template | 11 -- .../openshift-mirror-beta.repo.template | 6 - .../openshift-rhocp.repo.template | 11 -- .../group1/cos9-bootc-source.containerfile | 21 ++-- .../group1/rhel94-bootc-source.containerfile | 27 ++--- .../cos9-bootc-source-optionals.containerfile | 21 ++-- ...hel94-bootc-source-optionals.containerfile | 27 ++--- 9 files changed, 152 insertions(+), 90 deletions(-) delete mode 100644 test/bootc-sources/microshift-local.repo.template create mode 100644 test/bootc-sources/microshift_repo_config.sh.template delete mode 100644 test/bootc-sources/openshift-fast-datapath.repo.template delete mode 100644 test/bootc-sources/openshift-mirror-beta.repo.template delete mode 100644 test/bootc-sources/openshift-rhocp.repo.template diff --git a/test/bootc-sources/microshift-local.repo.template b/test/bootc-sources/microshift-local.repo.template deleted file mode 100644 index 9025341efe..0000000000 --- a/test/bootc-sources/microshift-local.repo.template +++ /dev/null @@ -1,6 +0,0 @@ -[microshift-local] -name=MicroShift Local Repository -baseurl=REPLACE_USHIFT_RPM_REPO_PATH -enabled=1 -gpgcheck=0 -skip_if_unavailable=0 diff --git a/test/bootc-sources/microshift_repo_config.sh.template b/test/bootc-sources/microshift_repo_config.sh.template new file mode 100644 index 0000000000..478d6b7d53 --- /dev/null +++ b/test/bootc-sources/microshift_repo_config.sh.template @@ -0,0 +1,112 @@ +#!/bin/bash +set -euo pipefail + +USHIFT_LOCAL_REPO_FILE=/etc/yum.repos.d/microshift-local.repo +OCP_MIRROR_REPO_FILE=/etc/yum.repos.d/openshift-mirror-beta.repo +OCP_DTPATH_REPO_FILE=/etc/yum.repos.d/openshift-fast-datapath.repo +OCP_RHOCP_REPO_FILE=/etc/yum.repos.d/openshift-rhocp.repo + +usage() { + echo "Usage: $(basename $0) <-create microshift_local_repo_path | -delete>" + [ -n "$1" ] && echo "ERROR: $1" + exit 1 +} + +del_repo_files() { + rm -f "${USHIFT_LOCAL_REPO_FILE}" + rm -f "${OCP_MIRROR_REPO_FILE}" + rm -f "${OCP_DTPATH_REPO_FILE}" + rm -f "${OCP_RHOCP_REPO_FILE}" +} + +config_local_repos() { + local -r repo_path=$1 + cat > "${USHIFT_LOCAL_REPO_FILE}" < "${OCP_MIRROR_REPO_FILE}" < "${OCP_DTPATH_REPO_FILE}" < "${OCP_RHOCP_REPO_FILE}" < Date: Thu, 9 May 2024 12:27:28 +0000 Subject: [PATCH 7/7] Add repository content printout --- test/bootc-sources/microshift_repo_config.sh.template | 9 ++++++--- .../layer5-bootc/group1/cos9-bootc-source.containerfile | 2 +- .../group1/rhel94-bootc-source.containerfile | 3 ++- .../group2/cos9-bootc-source-optionals.containerfile | 2 +- .../group2/rhel94-bootc-source-optionals.containerfile | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/bootc-sources/microshift_repo_config.sh.template b/test/bootc-sources/microshift_repo_config.sh.template index 478d6b7d53..34979671aa 100644 --- a/test/bootc-sources/microshift_repo_config.sh.template +++ b/test/bootc-sources/microshift_repo_config.sh.template @@ -92,16 +92,19 @@ case "${RUN_MODE}" in -create) LOCAL_REPO_PATH=$2 [ -z "${LOCAL_REPO_PATH}" ] && usage "MicroShift local repository path argument is missing" - + # Configure the local repositories config_local_repos "${LOCAL_REPO_PATH}" + # Configure the OS-specific remote repositories os_id=$(awk -F= '$1=="ID" { print $2 }' /etc/os-release | xargs) - if [ "${os_id}" == "centos" ] ; then + if [ "${os_id}" == "centos" ] ; then config_centos9_repos - elif [ "${os_id}" == "rhel" ] ; then + elif [ "${os_id}" == "rhel" ] ; then config_rhel9_repos else usage "Only RHEL or CentOS operating systems are supported" fi + # Print the repository contents + awk 'FNR==1 {print "=== " FILENAME " ==="} {print}' /etc/yum.repos.d/*.repo ;; -delete) del_repo_files diff --git a/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile index 8efd7257c6..1f8c3e94df 100644 --- a/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/cos9-bootc-source.containerfile @@ -18,7 +18,7 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH # is accessed by NetworkManager at /var/run/dbus. RUN [ ! -L /var/run ] && rm -rf /var/run && ln -s /run /var/ -# Add the following repositories: +# Add the following repositories and print their contents: # - MicroShift local RPM repository # - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies # Install MicroShift, few helper utilities and cleanup diff --git a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile index 205b473ada..68b1d22b8b 100644 --- a/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile +++ b/test/image-blueprints/layer5-bootc/group1/rhel94-bootc-source.containerfile @@ -18,11 +18,12 @@ COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH # is accessed by NetworkManager at /var/run/dbus. RUN [ ! -L /var/run ] && rm -rf /var/run && ln -s /run /var/ -# Add the following repositories: +# Add the following repositories and print their contents: # - MicroShift local RPM repository # - OpenShift previous minor version RHEL repositories for MicroShift dependencies # Install MicroShift, few helper utilities and cleanup RUN ${REPO_CONFIG_SCRIPT} -create ${USHIFT_RPM_REPO_PATH} && \ + awk 'FNR==1 {print "=== " FILENAME " ==="} {print}' /etc/yum.repos.d/*.repo && \ dnf install -y vi firewalld microshift && \ systemctl enable microshift && \ ${REPO_CONFIG_SCRIPT} -delete && \ diff --git a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile index a41f609068..c4ac8f94c8 100644 --- a/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/cos9-bootc-source-optionals.containerfile @@ -11,7 +11,7 @@ COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} # Copy the MicroShift repository contents COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# Add the following repositories: +# Add the following repositories and print their contents: # - MicroShift local RPM repository # - OpenShift Mirror Beta previous minor version repository for MicroShift dependencies # Install MicroShift optional packages and cleanup diff --git a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile index c05f4a0b40..561a49f6b9 100644 --- a/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile +++ b/test/image-blueprints/layer5-bootc/group2/rhel94-bootc-source-optionals.containerfile @@ -11,7 +11,7 @@ COPY --chmod=755 ./bootc-sources/microshift_repo_config.sh ${REPO_CONFIG_SCRIPT} # Copy the MicroShift repository contents COPY ./rpm-repos/$USHIFT_RPM_REPO_NAME $USHIFT_RPM_REPO_PATH -# Add the following repositories: +# Add the following repositories and print their contents: # - MicroShift local RPM repository # - OpenShift previous minor version RHEL repositories for MicroShift dependencies # Install MicroShift optional packages and cleanup