diff --git a/scripts/image-builder/cleanup.sh b/scripts/image-builder/cleanup.sh
index 204f0cca41..d2145e13cf 100755
--- a/scripts/image-builder/cleanup.sh
+++ b/scripts/image-builder/cleanup.sh
@@ -59,15 +59,17 @@ clean_osbuilder_services() {
     fi
 
     title "Stopping osbuild services"
-    sudo systemctl stop --now osbuild-composer.socket osbuild-composer.service
     for n in $(seq 100) ; do
         worker=osbuild-worker@${n}.service
         if sudo systemctl status "${worker}" &>/dev/null ; then
-            sudo systemctl stop --now "osbuild-worker@${n}.service"
+            sudo systemctl stop "osbuild-worker@${n}.service"
         else
             break
         fi
     done
+    # Don't stop the .socket as it can cause composer.service to fail
+    # stopping resulting in failed restarts which cause problem starting later.
+    sudo systemctl stop osbuild-composer.service
 
     title "Cleaning osbuild worker cache"
     sleep 5
@@ -79,10 +81,12 @@ restart_osbuilder_services() {
         return
     fi
 
-    if ! systemctl is-active -q osbuild-composer.socket &>/dev/null ; then
+    if ! systemctl is-active -q osbuild-composer.service &>/dev/null ; then
         title "Starting osbuild services"
-        sudo systemctl start osbuild-composer.socket
         sudo systemctl start osbuild-worker@1.service
+        # Thanks to osbuild-composer.socket, starting worker should be enough
+        # to start the composer, but left it just in case.
+        sudo systemctl start osbuild-composer.service
     fi
 }
 
diff --git a/test/bin/build_images.sh b/test/bin/build_images.sh
index 85e40b8ba9..0f88a04513 100755
--- a/test/bin/build_images.sh
+++ b/test/bin/build_images.sh
@@ -295,7 +295,23 @@ do_group() {
         echo "Building edge-commit from ${blueprint} ${parent_args}"
         # shellcheck disable=SC2086  # quote to avoid glob expansion
         build_cmd="sudo composer-cli compose start-ostree ${parent_args} --ref ${blueprint} ${blueprint} edge-commit"
-        buildid=$(${build_cmd} | awk '{print $2}')
+        for _ in $(seq 3); do
+            set +e
+            build_cmd_output=$(${build_cmd})
+            rc=$?
+            set -e
+            if [[ "${rc}" -eq 0 ]]; then
+                buildid=$(echo "${build_cmd_output}" | awk '/^Compose/ {print $2}')
+                break
+            fi
+            sleep 15
+        done
+
+        if [[ "${rc}" -ne 0 ]]; then
+            echo "Command failed consistently: ${build_cmd}"
+            exit 1
+        fi
+
         echo "Build ID ${buildid}"
         # Record a "build name" to be used as part of the unique
         # filename for the log we download next.
@@ -333,6 +349,12 @@ do_group() {
         builds_to_get=$(time "${SCRIPTDIR}/wait_images.py" "${buildid_list[@]}")
     fi
 
+    builds_to_get_num="$(echo "${builds_to_get}" | awk -F' ' '{print NF}')"
+    if [ "${#buildid_list[@]}" -ne "${builds_to_get_num}" ]; then
+        echo "wait_images.py returned unexpected amount of build IDs"
+        return 1
+    fi
+
     echo "Downloading build logs, metadata, and image"
     cd "${IMAGEDIR}/builds"
 
@@ -406,6 +428,7 @@ do_group() {
             record_junit "${groupdir}" "${alias_name}" "alias" "OK"
         else
             record_junit "${groupdir}" "${alias_name}" "alias" "FAILED"
+            return 1
         fi
     done
 
diff --git a/test/bin/wait_images.py b/test/bin/wait_images.py
index d289e0bd0d..b86f92d8cf 100755
--- a/test/bin/wait_images.py
+++ b/test/bin/wait_images.py
@@ -86,7 +86,14 @@ def restart_job(cmd):
     result = subprocess.run(cmd, shell=True, text=True, stdout=subprocess.PIPE)
     if result.returncode != 0:
         return ""
-    return result.stdout.split(" ")[1]
+    # Osbuild emits a warning on stdout about functionality we don't use in our blueprints,
+    # so we need to skip the lines we don't find interesting. Example output:
+    # > Warning: Please note that user customizations on "edge-commit" image type are deprecated and will be removed in the near future
+    # >
+    # > Compose 2c6a1ba2-4a18-49b8-a0f1-d103de1bd93a added to the queue
+    # Split output with \n and select line starting with "Compose", then split that line with space and select 2nd word.
+    line_with_id = next(line for line in result.stdout.split("\n") if line.startswith("Compose"))
+    return line_with_id.split(" ")[1]
 
 
 def copy_build_metadata(old_id, new_id):