From 535549c00e1535cb1e4031aedc32e6f7d4649550 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 26 Apr 2024 09:01:57 +0200 Subject: [PATCH 1/6] Fix restart_job so it's aware of warnings printed to stdout --- test/bin/wait_images.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/bin/wait_images.py b/test/bin/wait_images.py index d289e0bd0d..419b9ea991 100755 --- a/test/bin/wait_images.py +++ b/test/bin/wait_images.py @@ -86,7 +86,14 @@ def restart_job(cmd): result = subprocess.run(cmd, shell=True, text=True, stdout=subprocess.PIPE) if result.returncode != 0: return "" - return result.stdout.split(" ")[1] + # Osbuild emits a warning on stdout about functionality we don't use in our blueprints, + # so we need to skip the lines we don't find interesting. Example output: + # > Warning: Please note that user customizations on "edge-commit" image type are deprecated and will be removed in the near future + # > + # > Compose 2c6a1ba2-4a18-49b8-a0f1-d103de1bd93a added to the queue + # Split output with \n and select line starting with "Compose", then split that line with space and select 2nd word. + line_with_id = [line for line in result.stdout.split("\n") if line.startswith("Compose")][0] + return line_with_id.split(" ")[1] def copy_build_metadata(old_id, new_id): From d8d0cb6d0080fa8b093a72dd4c3db96814275c03 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 26 Apr 2024 14:35:35 +0200 Subject: [PATCH 2/6] Use generator instead of list comprehension Co-authored-by: Pablo Acevedo Montserrat --- test/bin/wait_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bin/wait_images.py b/test/bin/wait_images.py index 419b9ea991..b86f92d8cf 100755 --- a/test/bin/wait_images.py +++ b/test/bin/wait_images.py @@ -92,7 +92,7 @@ def restart_job(cmd): # > # > Compose 2c6a1ba2-4a18-49b8-a0f1-d103de1bd93a added to the queue # Split output with \n and select line starting with "Compose", then split that line with space and select 2nd word. - line_with_id = [line for line in result.stdout.split("\n") if line.startswith("Compose")][0] + line_with_id = next(line for line in result.stdout.split("\n") if line.startswith("Compose")) return line_with_id.split(" ")[1] From 1c4eae5c01894cd58007ada6fad1d610f7c2240e Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Tue, 23 Apr 2024 14:20:35 +0200 Subject: [PATCH 3/6] Don't stop the socket - it can cause composer to keep restarting and failing Disabling socket can cause composer to fail the shutdown and result in multiple restarts that fail and block later start (at the end of the script). Instead, stop workers first to keep composer from reactivating via socket, then stop the composer. --- scripts/image-builder/cleanup.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/image-builder/cleanup.sh b/scripts/image-builder/cleanup.sh index 204f0cca41..d2145e13cf 100755 --- a/scripts/image-builder/cleanup.sh +++ b/scripts/image-builder/cleanup.sh @@ -59,15 +59,17 @@ clean_osbuilder_services() { fi title "Stopping osbuild services" - sudo systemctl stop --now osbuild-composer.socket osbuild-composer.service for n in $(seq 100) ; do worker=osbuild-worker@${n}.service if sudo systemctl status "${worker}" &>/dev/null ; then - sudo systemctl stop --now "osbuild-worker@${n}.service" + sudo systemctl stop "osbuild-worker@${n}.service" else break fi done + # Don't stop the .socket as it can cause composer.service to fail + # stopping resulting in failed restarts which cause problem starting later. + sudo systemctl stop osbuild-composer.service title "Cleaning osbuild worker cache" sleep 5 @@ -79,10 +81,12 @@ restart_osbuilder_services() { return fi - if ! systemctl is-active -q osbuild-composer.socket &>/dev/null ; then + if ! systemctl is-active -q osbuild-composer.service &>/dev/null ; then title "Starting osbuild services" - sudo systemctl start osbuild-composer.socket sudo systemctl start osbuild-worker@1.service + # Thanks to osbuild-composer.socket, starting worker should be enough + # to start the composer, but left it just in case. + sudo systemctl start osbuild-composer.service fi } From 1a9762e964461553825879cb74ab286cf0e9b3f9 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 18 Apr 2024 12:58:08 +0200 Subject: [PATCH 4/6] When selecting compose ID select only valid line --- test/bin/build_images.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index dd2a7f4e2d..ba56055c4e 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -272,7 +272,23 @@ do_group() { echo "Building edge-commit from ${blueprint} ${parent_args}" # shellcheck disable=SC2086 # quote to avoid glob expansion build_cmd="sudo composer-cli compose start-ostree ${parent_args} --ref ${blueprint} ${blueprint} edge-commit" - buildid=$(${build_cmd} | awk '{print $2}') + for _ in $(seq 3); do + set +e + build_cmd_output=$(${build_cmd}) + rc=$? + set -e + if [[ "${rc}" -eq 0 ]]; then + buildid=$(echo "${build_cmd_output}" | awk '/^Compose/ {print $2}') + break + fi + sleep 15 + done + + if [[ "${rc}" -ne 0 ]]; then + echo "Command failed consistently: ${build_cmd}" + exit 1 + fi + echo "Build ID ${buildid}" # Record a "build name" to be used as part of the unique # filename for the log we download next. From f3c3364937a0726cdbe006ef45978bedc682ec2b Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 18 Apr 2024 13:34:50 +0200 Subject: [PATCH 5/6] Check if wait_images.py returned right amount of IDs --- test/bin/build_images.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index ba56055c4e..9e75a1d98a 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -317,6 +317,12 @@ do_group() { builds_to_get=$(time "${SCRIPTDIR}/wait_images.py" "${buildid_list[@]}") fi + builds_to_get_num="$(echo "${builds_to_get}" | awk -F' ' '{print NF}')" + if [ "${#buildid_list[@]}" -ne "${builds_to_get_num}" ]; then + echo "wait_images.py returned unexpected amount of build IDs" + return 1 + fi + echo "Downloading build logs, metadata, and image" cd "${IMAGEDIR}/builds" From 72eacbd50d5cc3d4d12dfefad954d490523f10bf Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 18 Apr 2024 14:31:14 +0200 Subject: [PATCH 6/6] Return 1 if aliasing failed --- test/bin/build_images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index 9e75a1d98a..0e5fae4911 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -396,6 +396,7 @@ do_group() { record_junit "${groupdir}" "${alias_name}" "alias" "OK" else record_junit "${groupdir}" "${alias_name}" "alias" "FAILED" + return 1 fi done