From 60a55b8b107fe81ee215403a5d6833a99a435910 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Wed, 18 Mar 2020 18:04:48 -0400 Subject: [PATCH 1/4] Retrieve installer sha from CI environment If we're in OpenShift CI in an installer PR, then the installer is built by rebasing on the target branch (i.e. master). The SHA that `openshift-install version` reports doesn't exist on GitHub. It's only in the environment that CI used to build the installer. This gets the PR SHA from the environment, and then fetchs the rhcos from that branch. There's a potential problem here if a PR branch is out of date. However, we really do need to get the rhcos.json from the PR branch in case it's being updated. The fixes for this are probably a bunch of API calls to see if the PR modifies rhcos.json, or actually check out the git repo which will increase the already long build time. --- rhcos.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/rhcos.sh b/rhcos.sh index 92c83ea7a..17eaeb00b 100644 --- a/rhcos.sh +++ b/rhcos.sh @@ -1,5 +1,17 @@ -# Get the git commit that the openshift installer was built from -OPENSHIFT_INSTALL_COMMIT=$($OPENSHIFT_INSTALLER version | grep commit | cut -d' ' -f4) +# If we're in OpenShift CI, get the sha for our PR. The one openshift-install version +# reports isn't correct due to how CI rebases the PR before building. +# +# FIXME(stbenjam): If a PR branch is not current with master, there is a +# potential for rhcos.json to be out of date. The potential solutions to +# this are not ideal, we'll either have to checkout the repo ourselves and +# check or make a bunch of API calls to see if the PR modified +# rhcos.json, otherwise just always get master. +if [[ "$JOB_NAME" =~ "openshift-installer" ]]; then + OPENSHIFT_INSTALL_COMMIT=${PULL_PULL_SHA:-$(echo "$JOB_SPEC" | jq -r '.refs.pulls[0].sha')} +else + # Get the git commit that the openshift installer was built from + OPENSHIFT_INSTALL_COMMIT=$($OPENSHIFT_INSTALLER version | grep commit | cut -d' ' -f4) +fi # Get the rhcos.json for that commit OPENSHIFT_INSTALLER_MACHINE_OS=${OPENSHIFT_INSTALLER_MACHINE_OS:-https://raw.githubusercontent.com/openshift/installer/$OPENSHIFT_INSTALL_COMMIT/data/data/rhcos.json} From dc5b175318056851db9ced05b859bfd5db321035 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Mon, 23 Mar 2020 08:16:54 -0400 Subject: [PATCH 2/4] Use alternative approach to extract from installer container --- 03_build_installer.sh | 1 + ocp_install_env.sh | 22 ++++++++++++++++++++++ rhcos.sh | 31 ++++++++++++++++--------------- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/03_build_installer.sh b/03_build_installer.sh index 63f9b5e7d..070358242 100755 --- a/03_build_installer.sh +++ b/03_build_installer.sh @@ -15,6 +15,7 @@ mkdir -p $OCP_DIR if [ -z "$KNI_INSTALL_FROM_GIT" ]; then # Extract openshift-install from the release image extract_installer "${OPENSHIFT_RELEASE_IMAGE}" $OCP_DIR + extract_rhcos_json "${OPENSHIFT_RELEASE_IMAGE}" $OCP_DIR else # Clone and build the installer from source clone_installer diff --git a/ocp_install_env.sh b/ocp_install_env.sh index 97540b615..a3c0f413e 100644 --- a/ocp_install_env.sh +++ b/ocp_install_env.sh @@ -39,6 +39,27 @@ function extract_installer() { extract_command openshift-baremetal-install "$1" "$2" } +function extract_rhcos_json() { + local release_image + local outdir + + release_image="$1" + outdir="$2" + pullsecret_file=$(mktemp "pullsecret--XXXXXXXXXX") + + echo "${PULL_SECRET}" > "${pullsecret_file}" + + baremetal_image=$(oc adm release info --image-for=baremetal-installer --registry-config "$pullsecret_file" "$release_image") + baremetal_container=$(podman create --authfile "$pullsecret_file" "$baremetal_image") + + # This is OK to fail as rhcos.json isn't available in every release, + # we'll download it from github if it's not available + podman cp "$baremetal_container":/var/cache/rhcos.json "$outdir" || true + + podman rm -f "$baremetal_container" + rm -rf "${pullsecret_file}" +} + function clone_installer() { # Clone repo, if not already present if [[ ! -d $OPENSHIFT_INSTALL_PATH ]]; then @@ -52,6 +73,7 @@ function build_installer() { cd $OPENSHIFT_INSTALL_PATH TAGS="libvirt baremetal" hack/build.sh popd + cp "$OPENSHIFT_INSTALL_PATH/data/data/rhcos.json" "$OCP_DIR" } # FIXME(stbenjam): This is not available in 4.3 (yet) diff --git a/rhcos.sh b/rhcos.sh index 17eaeb00b..6d3145eab 100644 --- a/rhcos.sh +++ b/rhcos.sh @@ -1,23 +1,24 @@ -# If we're in OpenShift CI, get the sha for our PR. The one openshift-install version -# reports isn't correct due to how CI rebases the PR before building. -# -# FIXME(stbenjam): If a PR branch is not current with master, there is a -# potential for rhcos.json to be out of date. The potential solutions to -# this are not ideal, we'll either have to checkout the repo ourselves and -# check or make a bunch of API calls to see if the PR modified -# rhcos.json, otherwise just always get master. -if [[ "$JOB_NAME" =~ "openshift-installer" ]]; then - OPENSHIFT_INSTALL_COMMIT=${PULL_PULL_SHA:-$(echo "$JOB_SPEC" | jq -r '.refs.pulls[0].sha')} +if [[ -f "$OCP_DIR/rhcos.json" ]]; then + MACHINE_OS_IMAGE_JSON=$(cat "$OCP_DIR/rhcos.json") else + + if [[ "$JOB_NAME" =~ "openshift-installer" ]]; then + # Get the SHA from the PR if we're in CI + OPENSHIFT_INSTALL_COMMIT=${PULL_PULL_SHA:-$(echo "$JOB_SPEC" | jq -r '.refs.pulls[0].sha')} + else + # Get the git commit that the openshift installer was built from + OPENSHIFT_INSTALL_COMMIT=$($OPENSHIFT_INSTALLER version | grep commit | cut -d' ' -f4) + fi + # Get the git commit that the openshift installer was built from OPENSHIFT_INSTALL_COMMIT=$($OPENSHIFT_INSTALLER version | grep commit | cut -d' ' -f4) -fi -# Get the rhcos.json for that commit -OPENSHIFT_INSTALLER_MACHINE_OS=${OPENSHIFT_INSTALLER_MACHINE_OS:-https://raw.githubusercontent.com/openshift/installer/$OPENSHIFT_INSTALL_COMMIT/data/data/rhcos.json} + # Get the rhcos.json for that commit + OPENSHIFT_INSTALLER_MACHINE_OS=${OPENSHIFT_INSTALLER_MACHINE_OS:-https://raw.githubusercontent.com/openshift/installer/$OPENSHIFT_INSTALL_COMMIT/data/data/rhcos.json} -# Get the rhcos.json for that commit, and find the baseURI and openstack image path -MACHINE_OS_IMAGE_JSON=$(curl "${OPENSHIFT_INSTALLER_MACHINE_OS}") + # Get the rhcos.json for that commit, and find the baseURI and openstack image path + MACHINE_OS_IMAGE_JSON=$(curl "${OPENSHIFT_INSTALLER_MACHINE_OS}") +fi export MACHINE_OS_INSTALLER_IMAGE_URL=$(echo "${MACHINE_OS_IMAGE_JSON}" | jq -r '.baseURI + .images.openstack.path') export MACHINE_OS_INSTALLER_IMAGE_SHA256=$(echo "${MACHINE_OS_IMAGE_JSON}" | jq -r '.images.openstack.sha256') From 61e078bc7537bb15cae72ac0784721b6b2e478b7 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Mon, 23 Mar 2020 14:31:48 -0400 Subject: [PATCH 3/4] Check if JOB_NAME is set --- rhcos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rhcos.sh b/rhcos.sh index 6d3145eab..fb6e6c67d 100644 --- a/rhcos.sh +++ b/rhcos.sh @@ -2,7 +2,7 @@ if [[ -f "$OCP_DIR/rhcos.json" ]]; then MACHINE_OS_IMAGE_JSON=$(cat "$OCP_DIR/rhcos.json") else - if [[ "$JOB_NAME" =~ "openshift-installer" ]]; then + if [[ -v JOB_NAME ]] && [[ "$JOB_NAME" =~ "openshift-installer" ]]; then # Get the SHA from the PR if we're in CI OPENSHIFT_INSTALL_COMMIT=${PULL_PULL_SHA:-$(echo "$JOB_SPEC" | jq -r '.refs.pulls[0].sha')} else From 9ab51f10c78cafb922226c9069b3343779f307a3 Mon Sep 17 00:00:00 2001 From: Stephen Benjamin Date: Mon, 23 Mar 2020 15:14:53 -0400 Subject: [PATCH 4/4] Fix whitespace in rhcos.sh --- rhcos.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rhcos.sh b/rhcos.sh index fb6e6c67d..5fed13bb6 100644 --- a/rhcos.sh +++ b/rhcos.sh @@ -2,13 +2,13 @@ if [[ -f "$OCP_DIR/rhcos.json" ]]; then MACHINE_OS_IMAGE_JSON=$(cat "$OCP_DIR/rhcos.json") else - if [[ -v JOB_NAME ]] && [[ "$JOB_NAME" =~ "openshift-installer" ]]; then - # Get the SHA from the PR if we're in CI - OPENSHIFT_INSTALL_COMMIT=${PULL_PULL_SHA:-$(echo "$JOB_SPEC" | jq -r '.refs.pulls[0].sha')} - else - # Get the git commit that the openshift installer was built from - OPENSHIFT_INSTALL_COMMIT=$($OPENSHIFT_INSTALLER version | grep commit | cut -d' ' -f4) - fi + if [[ -v JOB_NAME ]] && [[ "$JOB_NAME" =~ "openshift-installer" ]]; then + # Get the SHA from the PR if we're in CI + OPENSHIFT_INSTALL_COMMIT=${PULL_PULL_SHA:-$(echo "$JOB_SPEC" | jq -r '.refs.pulls[0].sha')} + else + # Get the git commit that the openshift installer was built from + OPENSHIFT_INSTALL_COMMIT=$($OPENSHIFT_INSTALLER version | grep commit | cut -d' ' -f4) + fi # Get the git commit that the openshift installer was built from OPENSHIFT_INSTALL_COMMIT=$($OPENSHIFT_INSTALLER version | grep commit | cut -d' ' -f4)