USHIFT-1953: improve scenario framework parallel output

test/bin/build_rpms.sh

@@ -37,22 +37,34 @@ BUILD_CMDS+=(
 )
 NUM_BUILD_CMDS="${#BUILD_CMDS[@]}"
 BUILD_RPMS_LOG="${IMAGEDIR}/build_rpms.json"
+BUILD_RPMS_JOB_LOG="${IMAGEDIR}/build_rpms_jobs.txt"
 mkdir -p "${IMAGEDIR}"
 
+# Show progress for interactive mode when stdin is a terminal
+if [ -t 0 ]; then
+    progress="--progress"
+else
+    progress=""
+fi
+
 # Disable the GNU Parallel citation
 echo will cite | parallel --citation &>/dev/null
 # Run the commands in parallel
-# Avoid --progress option because it requires tty and
-# it slows down execution in interactive shells
 echo "Starting parallel builds:"
 printf -- "  - %s\n" "${BUILD_CMDS[@]}"
 BUILD_OK=true
-if ! parallel --results "${BUILD_RPMS_LOG}" \
-        --jobs "${NUM_BUILD_CMDS}" \
-        ::: "${BUILD_CMDS[@]}" ; then
+if ! parallel \
+     ${progress} \
+     --results "${BUILD_RPMS_LOG}" \
+     --joblog "${BUILD_RPMS_JOB_LOG}" \
+     --jobs "${NUM_BUILD_CMDS}" \
+     ::: "${BUILD_CMDS[@]}" ; then
     BUILD_OK=false
 fi
 
+# Show the summary of the output of the parallel jobs.
+cat "${BUILD_RPMS_JOB_LOG}"
+
 if [ -f "${BUILD_RPMS_LOG}" ] ; then
     jq < "${BUILD_RPMS_LOG}"
 else

test/bin/ci_phase_iso_boot.sh

@@ -19,6 +19,8 @@ echo "Logging to ${LOGFILE}"
 # Set fd 1 and 2 to write to the log file
 exec &> >(tee >(awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush() }' >"${LOGFILE}"))
 
+LAUNCH_VMS_JOB_LOG="${IMAGEDIR}/launch_vm_jobs.txt"
+
 cd "${ROOTDIR}"
 
 # Make sure libvirtd is running. We do this here, because some of the
@@ -37,31 +39,28 @@ bash -x ./bin/manage_hypervisor_config.sh create
 # repository.
 bash -x ./bin/start_webserver.sh
 
-declare -A pidToScenario
-
-# Build all of the needed VMs
-for scenario in "${SCENARIO_SOURCES}"/*.sh; do
-    scenario_name=$(basename "${scenario}" .sh)
-    logfile="${SCENARIO_INFO_DIR}/${scenario_name}/boot.log"
-    mkdir -p "$(dirname "${logfile}")"
-    bash -x ./bin/scenario.sh create "${scenario}" >"${logfile}" 2>&1 &
-    pidToScenario["$!"]="${scenario}"
-    sleep 5
-done
-
-set +x
-for pid in "${!pidToScenario[@]}"; do echo "${pid} - ${pidToScenario[${pid}]}"; done
-set -x
-
-FAIL=0
-for job in $(jobs -p); do
-    jobs -l
-    echo "Waiting for job: ${job}"
-    if ! wait "${job}"; then
-        ((FAIL += 1))
-        echo "Failed to boot VMs for scenario: ${pidToScenario[${job}]}"
-    fi
-done
+# Show the summary of the output of the parallel jobs.
+if [ -t 0 ]; then
+    progress="--progress"
+else
+    progress=""
+fi
+
+# Tell scenario.sh to merge stderr into stdout
+export SCENARIO_MERGE_OUTPUT_STREAMS=true
+
+LAUNCH_OK=true
+if ! parallel \
+    ${progress} \
+    --results "${SCENARIO_INFO_DIR}/{/.}/boot.log" \
+    --joblog "${LAUNCH_VMS_JOB_LOG}" \
+    --delay 5 \
+    bash -x ./bin/scenario.sh create ::: "${SCENARIO_SOURCES}"/*.sh ; then
+   LAUNCH_OK=false
+fi
+
+# Show the summary of the output of the parallel jobs.
+cat "${LAUNCH_VMS_JOB_LOG}"
 
 echo "===================================="
 echo "System information after booting VMs"
@@ -72,9 +71,10 @@ sudo du -sk "${IMAGEDIR}"/* | sort -n
 sudo virsh list --all
 echo "===================================="
 
-if [ ${FAIL} -ne 0 ]; then
+if ! "${LAUNCH_OK}"; then
     echo "Failed to boot all VMs"
     exit 1
 fi
 
 echo "Boot phase complete"
+exit 0

test/bin/ci_phase_iso_build.sh

@@ -160,3 +160,4 @@ else
 fi
 
 echo "Build phase complete"
+exit 0

test/bin/ci_phase_test.sh

@@ -9,27 +9,38 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # shellcheck source=test/bin/common.sh
 source "${SCRIPTDIR}/common.sh"
 
+TEST_JOB_LOG="${IMAGEDIR}/test_jobs.txt"
+
 cd "${TESTDIR}"
 
 if [ ! -d "${RF_VENV}" ]; then
     "${ROOTDIR}/scripts/fetch_tools.sh" robotframework
 fi
 
-for scenario in "${SCENARIO_SOURCES}"/*.sh; do
-    scenario_name="$(basename "${scenario}" .sh)"
-    logfile="${SCENARIO_INFO_DIR}/${scenario_name}/run.log"
-    mkdir -p "$(dirname "${logfile}")"
-    SSH_PRIVATE_KEY="${HOME}/.ssh/id_rsa" bash -x ./bin/scenario.sh run "${scenario}" >"${logfile}" 2>&1 &
-done
+# Tell scenario.sh to merge stderr into stdout
+export SCENARIO_MERGE_OUTPUT_STREAMS=true
+
+# Show the summary of the output of the parallel jobs.
+if [ -t 0 ]; then
+    progress="--progress"
+else
+    progress=""
+fi
+
+TEST_OK=true
+if ! parallel \
+    ${progress} \
+    --results "${SCENARIO_INFO_DIR}/{/.}/run.log" \
+    --joblog "${TEST_JOB_LOG}" \
+    bash -x ./bin/scenario.sh run ::: "${SCENARIO_SOURCES}"/*.sh ; then
+   TEST_OK=false
+fi
 
-FAIL=0
-for job in $(jobs -p) ; do
-    jobs -l
-    echo "Waiting for job: ${job}"
-    wait "${job}" || ((FAIL+=1))
-done
+cat "${TEST_JOB_LOG}"
 
 echo "Test phase complete"
-if [[ ${FAIL} -ne 0 ]]; then
+if ! "${TEST_OK}"; then
+    echo "Some tests failed"
     exit 1
 fi
+exit 0

test/bin/scenario.sh

@@ -5,6 +5,11 @@
 
 set -euo pipefail
 
+SCENARIO_MERGE_OUTPUT_STREAMS=${SCENARIO_MERGE_OUTPUT_STREAMS:-false}
+if "${SCENARIO_MERGE_OUTPUT_STREAMS}"; then
+    exec 2>&1
+fi
+
 SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # shellcheck source=test/bin/common.sh
 source "${SCRIPTDIR}/common.sh"
@@ -269,49 +274,44 @@ wait_for_greenboot() {
 }
 
 start_junit() {
-    local outputfile="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/junit.xml"
-    mkdir -p "$(dirname "${outputfile}")"
+    mkdir -p "$(dirname "${JUNIT_OUTPUT_FILE}")"
 
-    echo "Creating ${outputfile}"
+    echo "Creating ${JUNIT_OUTPUT_FILE}"
 
-    cat - >"${outputfile}" <<EOF
+    cat - >"${JUNIT_OUTPUT_FILE}" <<EOF
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuite name="infrastructure for ${SCENARIO}" timestamp="$(date --iso-8601=ns)">
 EOF
 }
 
 close_junit() {
-    local outputfile="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/junit.xml"
-
-    echo '</testsuite>' >>"${outputfile}"
+    echo '</testsuite>' >>"${JUNIT_OUTPUT_FILE}"
 }
 
 record_junit() {
     local vmname="$1"
     local step="$2"
     local results="$3"
 
-    local outputfile="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/junit.xml"
-
-    cat - >>"${outputfile}" <<EOF
+    cat - >>"${JUNIT_OUTPUT_FILE}" <<EOF
 <testcase classname="${SCENARIO} ${vmname}" name="${step}">
 EOF
 
     case "${results}" in
         OK)
         ;;
         SKIP*)
-        cat - >>"${outputfile}" <<EOF
+        cat - >>"${JUNIT_OUTPUT_FILE}" <<EOF
 <skipped message="${results}" type="${step}-skipped" />
 EOF
         ;;
         *)
-        cat - >>"${outputfile}" <<EOF
+        cat - >>"${JUNIT_OUTPUT_FILE}" <<EOF
 <failure message="${results}" type="${step}-failure" />
 EOF
     esac
 
-    cat - >>"${outputfile}" <<EOF
+    cat - >>"${JUNIT_OUTPUT_FILE}" <<EOF
 </testcase>
 EOF
 }
@@ -565,17 +565,24 @@ run_tests() {
     local -r full_vmname="$(full_vm_name "${vmname}")"
     shift
 
+    start_junit
+    trap "close_junit" EXIT
+
     echo "Running tests with $# args" "$@"
 
     if [ ! -d "${RF_VENV}" ]; then
         error "RF_VENV (${RF_VENV}) does not exist, create it with: ${ROOTDIR}/scripts/fetch_tools.sh robotframework"
+        record_junit "${vmname}" "robot_framework_environment" "FAILED"
         exit 1
     fi
+    record_junit "${vmname}" "robot_framework_environment" "OK"
     local rf_binary="${RF_VENV}/bin/robot"
     if [ ! -f "${rf_binary}" ]; then
         error "robot is not installed to ${rf_binary}"
+        record_junit "${vmname}" "robot_framework_installed" "FAILED"
         exit 1
     fi
+    record_junit "${vmname}" "robot_framework_installed" "OK"
 
     # The IP file is created empty during the launch VM phase if the VM is has no NICs. This is the queue to skip
     # the variable file creation and greenboot check.
@@ -590,8 +597,10 @@ run_tests() {
             f="$(vm_property_filename "${vmname}" "${p}")"
             if [ ! -f "${f}" ]; then
                 error "Cannot read ${f}"
+                record_junit "${vmname}" "access_vm_property ${p}" "FAILED"
                 exit 1
             fi
+            record_junit "${vmname}" "access_vm_property ${p}" "OK"
         done
         local -r ssh_port=$(get_vm_property "${vmname}" "ssh_port")
         local -r api_port=$(get_vm_property "${vmname}" "api_port")
@@ -612,8 +621,10 @@ SSH_PRIV_KEY: "${SSH_PRIVATE_KEY:-}"
 SSH_PORT: ${ssh_port}
 EOF
         if ! wait_for_greenboot "${full_vmname}" "${vm_ip}"; then
+            record_junit "${vmname}" "pre_test_greenboot_check" "FAILED"
             return 1
         fi
+        record_junit "${vmname}" "pre_test_greenboot_check" "OK"
     fi
 
     local var_arg=${variable_file:+-V "${variable_file}"}
@@ -747,6 +758,7 @@ shift
 SCENARIO_SCRIPT="$(realpath "$1")"
 shift
 SCENARIO=$(basename "${SCENARIO_SCRIPT}" .sh)
+JUNIT_OUTPUT_FILE="${SCENARIO_INFO_DIR}/${SCENARIO}/phase_${action}/junit.xml"
 
 # Change directory to the test root
 cd "${SCRIPTDIR}/.."