Merge upstream and downstream specfiles at build time

packaging/microshift-builder.Containerfile

@@ -17,6 +17,8 @@ ARG USHIFT_PREBUILD_SCRIPT=/tmp/prebuild.sh
 ARG USHIFT_POSTBUILD_SCRIPT=/tmp/postbuild.sh
 ARG USHIFT_MODIFY_SPEC_SCRIPT=/tmp/modify-spec.py
 ARG USHIFT_BUILDRPMS_SCRIPT=/tmp/build-rpms.sh
+ARG SPECFILE_KINDNET=/tmp/kindnet.spec
+ARG SPECFILE_TOPOLVM=/tmp/topolvm.spec
 
 # Verify mandatory build arguments
 RUN if [ -z "${OKD_VERSION_TAG}" ] ; then \
@@ -46,37 +48,33 @@ RUN git clone --branch "${USHIFT_GITREF}" --single-branch "${USHIFT_GIT_URL}" "$
 
 WORKDIR ${HOME}/microshift/
 
-# Preparing the build scripts
 COPY --chmod=755 ./src/image/prebuild.sh ${USHIFT_PREBUILD_SCRIPT}
 RUN "${USHIFT_PREBUILD_SCRIPT}" --replace "${OKD_RELEASE_IMAGE}" "${OKD_VERSION_TAG}"
 
-COPY --chmod=755 ./src/image/build-rpms.sh ${USHIFT_BUILDRPMS_SCRIPT}
-# Modify the microshift.spec to remove packages not yet supported by the upstream.
-COPY --chmod=755 ./src/image/modify-spec.py ${USHIFT_MODIFY_SPEC_SCRIPT}
-# Disable the RPM and SRPM checks in the make-rpm.sh script.
-RUN sed -i -e 's,CHECK_RPMS="y",,g' -e 's,CHECK_SRPMS="y",,g' ./packaging/rpm/make-rpm.sh && \
-    ${USHIFT_MODIFY_SPEC_SCRIPT}
-
-# Building all MicroShift downstream RPMs and SRPMs
-# hadolint ignore=DL3059
-RUN "${USHIFT_BUILDRPMS_SCRIPT}" all
-
-# Building Kindnet upstream RPM
-COPY --chown=${USER}:${USER} ./src/kindnet/kindnet.spec "${HOME}/microshift/packaging/rpm/microshift.spec"
+COPY --chown=${USER}:${USER} ./src/kindnet/kindnet.spec "${SPECFILE_KINDNET}"
 COPY --chown=${USER}:${USER} ./src/kindnet/assets/  "${HOME}/microshift/assets/optional/"
 COPY --chown=${USER}:${USER} ./src/kindnet/dropins/ "${HOME}/microshift/packaging/kindnet/"
 COPY --chown=${USER}:${USER} ./src/kindnet/crio.conf.d/ "${HOME}/microshift/packaging/crio.conf.d/"
-# Prepare and build Kindnet upstream RPM
-RUN "${USHIFT_PREBUILD_SCRIPT}" --replace-kindnet "${OKD_RELEASE_IMAGE}" "${OKD_VERSION_TAG}" && \
-    "${USHIFT_BUILDRPMS_SCRIPT}" rpm
+RUN "${USHIFT_PREBUILD_SCRIPT}" --replace-kindnet "${OKD_RELEASE_IMAGE}" "${OKD_VERSION_TAG}"
 
-# Building TopoLVM upstream RPM
-COPY --chown=${USER}:${USER} ./src/topolvm/topolvm.spec "${HOME}/microshift/packaging/rpm/microshift.spec"
+COPY --chown=${USER}:${USER} ./src/topolvm/topolvm.spec "${SPECFILE_TOPOLVM}"
 COPY --chown=${USER}:${USER} ./src/topolvm/assets/  "${HOME}/microshift/assets/optional/topolvm/"
 COPY --chown=${USER}:${USER} ./src/topolvm/dropins/ "${HOME}/microshift/packaging/microshift/dropins/"
 COPY --chown=${USER}:${USER} ./src/topolvm/greenboot/ "${HOME}/microshift/packaging/greenboot/"
 COPY --chown=${USER}:${USER} ./src/topolvm/release/ "${HOME}/microshift/assets/optional/topolvm/"
-RUN "${USHIFT_BUILDRPMS_SCRIPT}" rpm
+
+COPY --chmod=755 ./src/image/modify-spec.py ${USHIFT_MODIFY_SPEC_SCRIPT}
+# Modify the microshift.spec:
+# - remove packages not yet supported by the upstream
+# - merge the kindnet.spec and topolvm.spec into the microshift.spec
+# Disable the RPM and SRPM checks in the make-rpm.sh script to not complain about removed packages
+RUN "${USHIFT_MODIFY_SPEC_SCRIPT}" ./packaging/rpm/microshift.spec "${SPECFILE_KINDNET}" "${SPECFILE_TOPOLVM}" && \
+    sed -i -e 's,CHECK_RPMS="y",,g' -e 's,CHECK_SRPMS="y",,g' ./packaging/rpm/make-rpm.sh
+
+# Build all MicroShift downstream RPMs and SRPMs
+COPY --chmod=755 ./src/image/build-rpms.sh ${USHIFT_BUILDRPMS_SCRIPT}
+# hadolint ignore=DL3059
+RUN "${USHIFT_BUILDRPMS_SCRIPT}" all
 
 # Post-build MicroShift configuration
 COPY --chmod=755 ./src/image/postbuild.sh ${USHIFT_POSTBUILD_SCRIPT}

src/image/modify-spec.py

@@ -7,6 +7,7 @@
 """
 
 import specfile
+import sys
 from itertools import product
 
 # Subpackages to remove
@@ -17,6 +18,7 @@
     'ai-model-serving',
     'cert-manager',
     'observability',
+    'sriov',
 ]
 
 # Sections to remove for the subpackages
@@ -30,35 +32,25 @@
 
 # Product of pkgs_to_remove and sections_to_remove, and the '-release-info' suffix.
 # Result is a list of strings like: 'package multus', 'description multus', 'files multus-release-info', etc.
-full_sections_to_remove = [ f"{p[0]} {p[1]}{p[2]}" for p in product(sections_to_remove, pkgs_to_remove, ["", "-release-info"]) ]
+full_sections_to_remove = [f"{p[0]} {p[1]}{p[2]}" for p in product(sections_to_remove, pkgs_to_remove, ["", "-release-info"])]
 
 # Words that identify lines to remove from the install section.
 install_keywords_to_remove = [
-    'multus',
-    'low-latency',
+    *pkgs_to_remove,
     'lib/tuned',
     '05-high-performance-runtime.conf',
     'microshift-baseline',
     'microshift-tuned',
-    'gateway-api',
-    'ai-model-serving',
-    'cert-manager',
-    'observability',
 ]
 
-s = specfile.Specfile(
-    './packaging/rpm/microshift.spec',
-    # Dummy macro values - they are referenced in the spec file but not provided by default, only added when running make-rpm.sh.
-    # Without these, specfile cannot be parsed because of the recursive references (because microshift.spec actually redefines macros defined by default...)
-    macros=[('release', '1'), ('version', '4.0.0'), ('commit', 'x'), ('embedded_git_tag', 'tag'), ('embedded_git_tree_state', 'clean')])
 
-with s.sections() as sections:
+def remove_downstream_unsupported_packages(sections):
     for id in full_sections_to_remove:
         try:
             sec = sections.get(id)
             sections.remove(sec)
             print(f"Removing section: '%{id}'")
-        except ValueError as e:
+        except ValueError:
             # Ignore non-existent sections
             pass
 
@@ -83,4 +75,46 @@
     i.clear()
     i.extend(new_install)
 
-s.save()
+
+def merge_specfile(sections, extra_sections):
+    for extra_section in extra_sections:
+        if extra_section.id == 'install':
+            continue
+        if extra_section.id.startswith(('files ', 'description ', 'package ')):
+            # Add before the section that precedes the changelog to keep the changelog 'usage' comment in right place
+            print(f"Adding section: '{extra_section.id}' to MicroShift downstream specfile")
+            sections.insert(len(sections) - 3, extra_section)
+
+    microshift_installs = sections.install
+    extra_installs = extra_sections.install
+    print(f"Adding following content to the MicroShift downstream specfile install section:\n{extra_installs}")
+    for line in extra_installs:
+        microshift_installs.append(line)
+
+
+def open_specfile(path):
+    return specfile.Specfile(
+        path,
+        # Dummy macro values - they are referenced in the spec file but not provided by default, only added when running make-rpm.sh.
+        # Without these, specfile cannot be parsed because of the recursive references (because microshift.spec actually redefines macros defined by default...)
+        macros=[('release', '1'), ('version', '4.0.0'), ('commit', 'x'), ('embedded_git_tag', 'tag'), ('embedded_git_tree_state', 'clean')])
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print(f"Usage: {sys.argv[0]} <primary_specfile> <specfile_to_merge>...", file=sys.stderr)
+        sys.exit(1)
+    specfiles = sys.argv[1:]
+    primary_specfile = specfiles[0]
+    specfiles_to_merge = specfiles[1:]
+
+    microshift_specfile = open_specfile(primary_specfile)
+    with microshift_specfile.sections() as microshift_sections:
+        remove_downstream_unsupported_packages(microshift_sections)
+
+        for specfile_to_merge in specfiles_to_merge:
+            print(f"Merging specfile: {specfile_to_merge} to MicroShift downstream specfile")
+            with open_specfile(specfile_to_merge).sections() as extra_sections:
+                merge_specfile(microshift_sections, extra_sections)
+
+    microshift_specfile.save()

src/topolvm/topolvm.spec

@@ -52,6 +52,7 @@ release. These files contain the list of container image references used by the
 %{_datadir}/microshift/release/release-topolvm-{x86_64,aarch64}.json
 
 %install
+# topolvm
 install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/001-microshift-topolvm
 install -p -m644 assets/optional/topolvm/*.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/001-microshift-topolvm
 
@@ -61,5 +62,6 @@ install -p -m644 packaging/microshift/dropins/disable-storage-csi.yaml %{buildro
 install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/check/required.d
 install -p -m755 packaging/greenboot/microshift-topolvm-check.sh %{buildroot}%{_sysconfdir}/greenboot/check/required.d/50_microshift_topolvm_check.sh
 
+# topolvm-release-info
 install -d -m755 %{buildroot}%{_datadir}/microshift/release
 install -p -m644 assets/optional/topolvm/release-topolvm-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release