diff --git a/scripts/image-builder/cleanup.sh b/scripts/image-builder/cleanup.sh
index 197745632d..d2145e13cf 100755
--- a/scripts/image-builder/cleanup.sh
+++ b/scripts/image-builder/cleanup.sh
@@ -59,15 +59,17 @@ clean_osbuilder_services() {
     fi
 
     title "Stopping osbuild services"
-    sudo systemctl stop --now osbuild-composer.socket osbuild-composer.service
     for n in $(seq 100) ; do
         worker=osbuild-worker@${n}.service
         if sudo systemctl status "${worker}" &>/dev/null ; then
-            sudo systemctl stop --now "osbuild-worker@${n}.service"
+            sudo systemctl stop "osbuild-worker@${n}.service"
         else
             break
         fi
     done
+    # Don't stop the .socket as it can cause composer.service to fail
+    # stopping resulting in failed restarts which cause problem starting later.
+    sudo systemctl stop osbuild-composer.service
 
     title "Cleaning osbuild worker cache"
     sleep 5
@@ -79,19 +81,12 @@ restart_osbuilder_services() {
         return
     fi
 
-    if ! systemctl is-active -q osbuild-composer.socket &>/dev/null ; then
+    if ! systemctl is-active -q osbuild-composer.service &>/dev/null ; then
         title "Starting osbuild services"
-        sudo systemctl start osbuild-composer.socket
-        for try in $(seq 3); do
-            sleep 1 # Give composer some time to be ready for workers
-            if sudo systemctl start osbuild-worker@1.service; then
-                break
-            else
-                if [ "${try}" -eq 2 ]; then
-                    return 1
-                fi
-            fi
-        done
+        sudo systemctl start osbuild-worker@1.service
+        # Thanks to osbuild-composer.socket, starting worker should be enough
+        # to start the composer, but left it just in case.
+        sudo systemctl start osbuild-composer.service
     fi
 }