diff --git a/ci/Dockerfile b/ci/Dockerfile
index 67a5e7a6d..913096e1e 100644
--- a/ci/Dockerfile
+++ b/ci/Dockerfile
@@ -8,5 +8,17 @@ RUN go test -v -c -o layering_test ./tests/layering
 
 FROM registry.svc.ci.openshift.org/coreos/coreos-assembler:latest
 WORKDIR /src
+ENV COSA_DIR=/tmp/cosa
+# Prow doesn't support emptydir for jobs today
+ENV COSA_SKIP_OVERLAY=1
 COPY --from=builder /go/src/github.com/openshift/os/layering_test /usr/local/bin/layering_test
 COPY . .
+RUN mkdir -p "${COSA_DIR}"
+# We need to make sure that root can read / write to the COSA_DIR so that
+# when this container is actually run, we have permissions to read and
+# write to the COSA_DIR to allow the Kola tests to run.
+USER root
+RUN chgrp -Rf root "${COSA_DIR}" && \
+  chmod -Rf g+w "${COSA_DIR}"
+USER builder
+WORKDIR /tmp/cosa
diff --git a/ci/prow-build-test-qemu.sh b/ci/prow-build-test-qemu.sh
index 883dd2edd..11b7a32e0 100755
--- a/ci/prow-build-test-qemu.sh
+++ b/ci/prow-build-test-qemu.sh
@@ -48,7 +48,9 @@ cosa kola --basic-qemu-scenarios
 kola run-upgrade -b rhcos -v --find-parent-image --qemu-image-dir tmp/ --output-dir tmp/kola-upgrade
 cosa kola run --parallel 2
 # Build metal + installer now so we can test them
-cosa buildextend-metal && cosa buildextend-metal4k && cosa buildextend-live
+cosa buildextend-metal
+cosa buildextend-metal4k
+cosa buildextend-live
 # compress the metal and metal4k images now so we're testing
 # installs with the image format we ship
 cosa compress --artifact=metal --artifact=metal4k