From 02f1768d9c211b9f80af70d66048e04948afb9fb Mon Sep 17 00:00:00 2001 From: Zack Zlotnik Date: Mon, 6 Jun 2022 15:15:45 -0400 Subject: [PATCH] refactors cosa session initialization This ensures that regardless of entrypoint, the cosa session is always initialized the same way. This also separates the cosa session initialization steps from the cosa build steps to aid in speeding up CI. --- ci/build-test-qemu.sh | 8 +++--- ci/cosa-build.sh | 6 +++++ ci/prow-build-test-qemu.sh | 45 +++---------------------------- ci/prow-build.sh | 55 +++++--------------------------------- ci/prow-prepare.sh | 49 +++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 94 deletions(-) create mode 100755 ci/cosa-build.sh create mode 100755 ci/prow-prepare.sh diff --git a/ci/build-test-qemu.sh b/ci/build-test-qemu.sh index 5c645cd18..f6cf25cf4 100755 --- a/ci/build-test-qemu.sh +++ b/ci/build-test-qemu.sh @@ -5,8 +5,6 @@ dn=$(dirname $0) # Prow jobs don't support adding emptydir today export COSA_SKIP_OVERLAY=1 # Create a temporary cosa workdir if COSA_DIR is not set. -cosa_dir="${COSA_DIR:-$(mktemp -d)}" -echo "Using $cosa_dir for build" -cd "$cosa_dir" -cosa init --transient /src -exec ${dn}/prow-build-test-qemu.sh +export COSA_DIR="${COSA_DIR:-$(mktemp -d)}" +cd "$COSA_DIR" +exec "${dn}/prow-build-test-qemu.sh" diff --git a/ci/cosa-build.sh b/ci/cosa-build.sh new file mode 100755 index 000000000..ed5d4ee4f --- /dev/null +++ b/ci/cosa-build.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -xeuo pipefail + +# This script assumes that `cosa init` has been run. +cosa build +cosa buildextend-extensions diff --git a/ci/prow-build-test-qemu.sh b/ci/prow-build-test-qemu.sh index 883dd2edd..98581feb8 100755 --- a/ci/prow-build-test-qemu.sh +++ b/ci/prow-build-test-qemu.sh @@ -1,49 +1,12 @@ #!/bin/bash set -xeuo pipefail -# This script is called via build-test-qemu.sh which is the main Prow -# entrypoint for PRs to this repo, as well as for PRs on other repos, -# mainly coreos-assembler. It assumes that `cosa init` has been run. -REDIRECTOR_URL="https://rhcos-redirector.apps.art.xq1c.p1.openshiftapps.com/art/storage/releases/" +dn="$(dirname "$0")" -# record information about cosa + rpm-ostree -if test -d /cosa; then - jq . < /cosa/coreos-assembler-git.json -fi -rpm-ostree --version +# Create a temporary cosa workdir if COSA_DIR is not set. +export COSA_DIR="${COSA_DIR:-$(mktemp -d)}" -# We generate .repo files which write to the source, but -# we captured the source as part of the Docker build. -# In OpenShift default SCC we'll run as non-root, so we need -# to make a new copy of the source. TODO fix cosa to be happy -# if src/config already exists instead of wanting to reference -# it or clone it. Or we could write our .repo files to a separate -# place. -if test '!' -w src/config; then - git clone --recurse src/config src/config.writable - rm src/config -rf - mv src/config.writable src/config -fi - -# -# NOTE: If you are adjusting how the repos are fetched in this script, you -# must also make the same change in the `prow-build.sh` script -# -# Grab the raw value of `mutate-os-release` and use sed to convert the value -# to X-Y format -ocpver=$(rpm-ostree compose tree --print-only src/config/manifest.yaml | jq -r '.["mutate-os-release"]') -ocpver_mut=$(rpm-ostree compose tree --print-only src/config/manifest.yaml | jq -r '.["mutate-os-release"]' | sed 's|\.|-|') -prev_build_url=${REDIRECTOR_URL}/rhcos-${ocpver}/ -# we want to use RHEL 8.5 for testing until we can start using 8.6 -# see https://github.com/openshift/release/pull/26193 -curl -L http://base-"${ocpver_mut}"-rhel85.ocp.svc.cluster.local > src/config/ocp.repo -# fetch the 8.6 appstream repo to enable building of extensions -# see: https://github.com/openshift/os/issues/795 -curl -Ls http://base-"${ocpver_mut}"-rhel86.ocp.svc.cluster.local | grep -A 3 rhel-8-appstream | sed '1,3 s/rhel-8-appstream/rhel-86-appstream/g' >> src/config/ocp.repo -cosa buildfetch --url=${prev_build_url} -cosa fetch -cosa build -cosa buildextend-extensions +"$dn/prow-build.sh" cosa kola --basic-qemu-scenarios kola run-upgrade -b rhcos -v --find-parent-image --qemu-image-dir tmp/ --output-dir tmp/kola-upgrade cosa kola run --parallel 2 diff --git a/ci/prow-build.sh b/ci/prow-build.sh index 99a0381b7..1e905bcc4 100755 --- a/ci/prow-build.sh +++ b/ci/prow-build.sh @@ -1,53 +1,12 @@ #!/bin/bash set -xeuo pipefail -# Prow jobs don't support adding emptydir today -export COSA_SKIP_OVERLAY=1 -# Create a temporary cosa workdir if COSA_DIR is not set. -cosa_dir="${COSA_DIR:-$(mktemp -d)}" -echo "Using $cosa_dir for build" -cd "$cosa_dir" -cosa init --transient /src - -# This script is called via build.sh which is the main Prow -# entrypoint for PRs to this repo, as well as for PRs on other repos, -# mainly coreos-assembler. It assumes that `cosa init` has been run. - -REDIRECTOR_URL="https://rhcos-redirector.apps.art.xq1c.p1.openshiftapps.com/art/storage/releases/" - -# record information about cosa + rpm-ostree -if test -d /cosa; then - jq . < /cosa/coreos-assembler-git.json -fi -rpm-ostree --version +dn="$(dirname "$0")" -# We generate .repo files which write to the source, but -# we captured the source as part of the Docker build. -# In OpenShift default SCC we'll run as non-root, so we need -# to make a new copy of the source. TODO fix cosa to be happy -# if src/config already exists instead of wanting to reference -# it or clone it. Or we could write our .repo files to a separate -# place. -if test '!' -w src/config; then - git clone --recurse src/config src/config.writable - rm src/config -rf - mv src/config.writable src/config -fi - -# -# NOTE: If you are adjusting how the repos are fetched in this script, you -# must also make the same change in the `prow-build-test-qemu.sh` script -# -# Grab the raw value of `mutate-os-release` and use sed to convert the value -# to X-Y format -ocpver=$(rpm-ostree compose tree --print-only src/config/manifest.yaml | jq -r '.["mutate-os-release"]') -ocpver_mut=$(rpm-ostree compose tree --print-only src/config/manifest.yaml | jq -r '.["mutate-os-release"]' | sed 's|\.|-|') -prev_build_url=${REDIRECTOR_URL}/rhcos-${ocpver}/ - -# Fetch RHEL 8.6 repos -curl -L http://base-"${ocpver_mut}"-rhel86.ocp.svc.cluster.local > src/config/ocp.repo +# Create a temporary cosa workdir if COSA_DIR is not set. +export COSA_DIR="${COSA_DIR:-$(mktemp -d)}" +echo "Using $COSA_DIR for build" +cd "$COSA_DIR" -cosa buildfetch --url=${prev_build_url} -cosa fetch -cosa build -cosa buildextend-extensions +"$dn/prow-prepare.sh" +"$dn/cosa-build.sh" diff --git a/ci/prow-prepare.sh b/ci/prow-prepare.sh new file mode 100755 index 000000000..27687010e --- /dev/null +++ b/ci/prow-prepare.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -xeuo pipefail + +# This script sets up the cosa session for a given Prow CI run. + +# Prow jobs don't support adding emptydir today +export COSA_SKIP_OVERLAY=1 +cosa init --transient /src + +# This script is called via prow-build.sh scripts which is the main Prow +# entrypoint for PRs to this repo, as well as for PRs on other repos, +# mainly coreos-assembler. It assumes that `cosa init` has been run. + +REDIRECTOR_URL="https://rhcos-redirector.apps.art.xq1c.p1.openshiftapps.com/art/storage/releases/" + +# record information about cosa + rpm-ostree +if test -d /cosa; then + jq . < /cosa/coreos-assembler-git.json +fi +rpm-ostree --version + +# We generate .repo files which write to the source, but +# we captured the source as part of the Docker build. +# In OpenShift default SCC we'll run as non-root, so we need +# to make a new copy of the source. TODO fix cosa to be happy +# if src/config already exists instead of wanting to reference +# it or clone it. Or we could write our .repo files to a separate +# place. +if test '!' -w src/config; then + git clone --recurse src/config src/config.writable + rm src/config -rf + mv src/config.writable src/config +fi + +# +# NOTE: If you are adjusting how the repos are fetched in this script, you +# must also make the same change in the `prow-build-test-qemu.sh` script +# +# Grab the raw value of `mutate-os-release` and use sed to convert the value +# to X-Y format +ocpver=$(rpm-ostree compose tree --print-only src/config/manifest.yaml | jq -r '.["mutate-os-release"]') +ocpver_mut=$(rpm-ostree compose tree --print-only src/config/manifest.yaml | jq -r '.["mutate-os-release"]' | sed 's|\.|-|') +prev_build_url=${REDIRECTOR_URL}/rhcos-${ocpver}/ + +# Fetch RHEL 8.6 repos +curl -L http://base-"${ocpver_mut}"-rhel86.ocp.svc.cluster.local > src/config/ocp.repo + +cosa buildfetch --url=${prev_build_url} +cosa fetch