From 3709d59fdf93ca66281fd219b47344714fb3bb6e Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 26 Apr 2024 09:01:57 +0200 Subject: [PATCH 1/6] Fix restart_job so it's aware of warnings printed to stdout --- test/bin/wait_images.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/bin/wait_images.py b/test/bin/wait_images.py index d289e0bd0d..419b9ea991 100755 --- a/test/bin/wait_images.py +++ b/test/bin/wait_images.py @@ -86,7 +86,14 @@ def restart_job(cmd): result = subprocess.run(cmd, shell=True, text=True, stdout=subprocess.PIPE) if result.returncode != 0: return "" - return result.stdout.split(" ")[1] + # Osbuild emits a warning on stdout about functionality we don't use in our blueprints, + # so we need to skip the lines we don't find interesting. Example output: + # > Warning: Please note that user customizations on "edge-commit" image type are deprecated and will be removed in the near future + # > + # > Compose 2c6a1ba2-4a18-49b8-a0f1-d103de1bd93a added to the queue + # Split output with \n and select line starting with "Compose", then split that line with space and select 2nd word. + line_with_id = [line for line in result.stdout.split("\n") if line.startswith("Compose")][0] + return line_with_id.split(" ")[1] def copy_build_metadata(old_id, new_id): From 11fc55e379eeb15f076f40642162de801da3f941 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Fri, 26 Apr 2024 14:35:35 +0200 Subject: [PATCH 2/6] Use generator instead of list comprehension Co-authored-by: Pablo Acevedo Montserrat --- test/bin/wait_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bin/wait_images.py b/test/bin/wait_images.py index 419b9ea991..b86f92d8cf 100755 --- a/test/bin/wait_images.py +++ b/test/bin/wait_images.py @@ -92,7 +92,7 @@ def restart_job(cmd): # > # > Compose 2c6a1ba2-4a18-49b8-a0f1-d103de1bd93a added to the queue # Split output with \n and select line starting with "Compose", then split that line with space and select 2nd word. - line_with_id = [line for line in result.stdout.split("\n") if line.startswith("Compose")][0] + line_with_id = next(line for line in result.stdout.split("\n") if line.startswith("Compose")) return line_with_id.split(" ")[1] From 7993af1220a3f5d18f9788d3c907093f55215a36 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Tue, 23 Apr 2024 14:20:35 +0200 Subject: [PATCH 3/6] Don't stop the socket - it can cause composer to keep restarting and failing Disabling socket can cause composer to fail the shutdown and result in multiple restarts that fail and block later start (at the end of the script). Instead, stop workers first to keep composer from reactivating via socket, then stop the composer. --- scripts/image-builder/cleanup.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/image-builder/cleanup.sh b/scripts/image-builder/cleanup.sh index 204f0cca41..d2145e13cf 100755 --- a/scripts/image-builder/cleanup.sh +++ b/scripts/image-builder/cleanup.sh @@ -59,15 +59,17 @@ clean_osbuilder_services() { fi title "Stopping osbuild services" - sudo systemctl stop --now osbuild-composer.socket osbuild-composer.service for n in $(seq 100) ; do worker=osbuild-worker@${n}.service if sudo systemctl status "${worker}" &>/dev/null ; then - sudo systemctl stop --now "osbuild-worker@${n}.service" + sudo systemctl stop "osbuild-worker@${n}.service" else break fi done + # Don't stop the .socket as it can cause composer.service to fail + # stopping resulting in failed restarts which cause problem starting later. + sudo systemctl stop osbuild-composer.service title "Cleaning osbuild worker cache" sleep 5 @@ -79,10 +81,12 @@ restart_osbuilder_services() { return fi - if ! systemctl is-active -q osbuild-composer.socket &>/dev/null ; then + if ! systemctl is-active -q osbuild-composer.service &>/dev/null ; then title "Starting osbuild services" - sudo systemctl start osbuild-composer.socket sudo systemctl start osbuild-worker@1.service + # Thanks to osbuild-composer.socket, starting worker should be enough + # to start the composer, but left it just in case. + sudo systemctl start osbuild-composer.service fi } From b9bc39eca281ae130aefbf97e51942f787d643b4 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 18 Apr 2024 12:58:08 +0200 Subject: [PATCH 4/6] When selecting compose ID select only valid line --- test/bin/build_images.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index 85e40b8ba9..f92e64fc7c 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -295,7 +295,23 @@ do_group() { echo "Building edge-commit from ${blueprint} ${parent_args}" # shellcheck disable=SC2086 # quote to avoid glob expansion build_cmd="sudo composer-cli compose start-ostree ${parent_args} --ref ${blueprint} ${blueprint} edge-commit" - buildid=$(${build_cmd} | awk '{print $2}') + for _ in $(seq 3); do + set +e + build_cmd_output=$(${build_cmd}) + rc=$? + set -e + if [[ "${rc}" -eq 0 ]]; then + buildid=$(echo "${build_cmd_output}" | awk '/^Compose/ {print $2}') + break + fi + sleep 15 + done + + if [[ "${rc}" -ne 0 ]]; then + echo "Command failed consistently: ${build_cmd}" + exit 1 + fi + echo "Build ID ${buildid}" # Record a "build name" to be used as part of the unique # filename for the log we download next. From 8c8e4815e37166d318fdceae994e6895101c31de Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 18 Apr 2024 13:34:50 +0200 Subject: [PATCH 5/6] Check if wait_images.py returned right amount of IDs --- test/bin/build_images.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index f92e64fc7c..b8c094e9e1 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -349,6 +349,12 @@ do_group() { builds_to_get=$(time "${SCRIPTDIR}/wait_images.py" "${buildid_list[@]}") fi + builds_to_get_num="$(echo "${builds_to_get}" | awk -F' ' '{print NF}')" + if [ "${#buildid_list[@]}" -ne "${builds_to_get_num}" ]; then + echo "wait_images.py returned unexpected amount of build IDs" + return 1 + fi + echo "Downloading build logs, metadata, and image" cd "${IMAGEDIR}/builds" From a6ac5fe98e3afb9ef6f651f4ab8c507ff6eeb1be Mon Sep 17 00:00:00 2001 From: Patryk Matuszak Date: Thu, 18 Apr 2024 14:31:14 +0200 Subject: [PATCH 6/6] Return 1 if aliasing failed --- test/bin/build_images.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh index b8c094e9e1..0f88a04513 100755 --- a/test/bin/build_images.sh +++ b/test/bin/build_images.sh @@ -428,6 +428,7 @@ do_group() { record_junit "${groupdir}" "${alias_name}" "alias" "OK" else record_junit "${groupdir}" "${alias_name}" "alias" "FAILED" + return 1 fi done