From 8661d1e690ccec45b116f79f40c235d6012cabc6 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 28 Nov 2023 16:46:30 -0500 Subject: [PATCH 1/8] add human readable summary of parallel rpm build commands --- test/bin/build_rpms.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/bin/build_rpms.sh b/test/bin/build_rpms.sh index 1025abe41c..ae395eec33 100755 --- a/test/bin/build_rpms.sh +++ b/test/bin/build_rpms.sh @@ -37,6 +37,7 @@ BUILD_CMDS+=( ) NUM_BUILD_CMDS="${#BUILD_CMDS[@]}" BUILD_RPMS_LOG="${IMAGEDIR}/build_rpms.json" +BUILD_RPMS_JOB_LOG="${IMAGEDIR}/build_rpms_jobs.txt" mkdir -p "${IMAGEDIR}" # Disable the GNU Parallel citation @@ -48,11 +49,15 @@ echo "Starting parallel builds:" printf -- " - %s\n" "${BUILD_CMDS[@]}" BUILD_OK=true if ! parallel --results "${BUILD_RPMS_LOG}" \ + --joblog "${BUILD_RPMS_JOB_LOG}" \ --jobs "${NUM_BUILD_CMDS}" \ ::: "${BUILD_CMDS[@]}" ; then BUILD_OK=false fi +# Show the summary of the output of the parallel jobs. +cat "${BUILD_RPMS_JOB_LOG}" + if [ -f "${BUILD_RPMS_LOG}" ] ; then jq < "${BUILD_RPMS_LOG}" else From 0c0b1f59bf2069ee78ac7d6b4abc676bffe9a10f Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 29 Nov 2023 08:43:05 -0500 Subject: [PATCH 2/8] show progress when running build_rpms.sh interactively When stdin is a tty, add --progress to the parallel command used to run the builds. --- test/bin/build_rpms.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/bin/build_rpms.sh b/test/bin/build_rpms.sh index ae395eec33..e1c41ddd63 100755 --- a/test/bin/build_rpms.sh +++ b/test/bin/build_rpms.sh @@ -40,18 +40,25 @@ BUILD_RPMS_LOG="${IMAGEDIR}/build_rpms.json" BUILD_RPMS_JOB_LOG="${IMAGEDIR}/build_rpms_jobs.txt" mkdir -p "${IMAGEDIR}" +# Show progress for interactive mode when stdin is a terminal +if [ -t 0 ]; then + progress="--progress" +else + progress="" +fi + # Disable the GNU Parallel citation echo will cite | parallel --citation &>/dev/null # Run the commands in parallel -# Avoid --progress option because it requires tty and -# it slows down execution in interactive shells echo "Starting parallel builds:" printf -- " - %s\n" "${BUILD_CMDS[@]}" BUILD_OK=true -if ! parallel --results "${BUILD_RPMS_LOG}" \ - --joblog "${BUILD_RPMS_JOB_LOG}" \ - --jobs "${NUM_BUILD_CMDS}" \ - ::: "${BUILD_CMDS[@]}" ; then +if ! parallel \ + ${progress} \ + --results "${BUILD_RPMS_LOG}" \ + --joblog "${BUILD_RPMS_JOB_LOG}" \ + --jobs "${NUM_BUILD_CMDS}" \ + ::: "${BUILD_CMDS[@]}" ; then BUILD_OK=false fi From b0e0d86da75339dd43d3c920b5354af6c0a37418 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 28 Nov 2023 18:10:42 -0500 Subject: [PATCH 3/8] let a caller merge the output streams of scenario.sh This makes it easier to use parallel to invoke scenario.sh with bash -x and have the stderr and stdout messages logged to the same file for debugging. --- test/bin/scenario.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 044dd99361..007d2318a0 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -5,6 +5,11 @@ set -euo pipefail +SCENARIO_MERGE_OUTPUT_STREAMS=${SCENARIO_MERGE_OUTPUT_STREAMS:-false} +if "${SCENARIO_MERGE_OUTPUT_STREAMS}"; then + exec 2>&1 +fi + SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=test/bin/common.sh source "${SCRIPTDIR}/common.sh" From d41ac0d4c9746f0455a39cb2f959d61b11cc66ce Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 28 Nov 2023 18:11:42 -0500 Subject: [PATCH 4/8] use parallel to boot scenario vms Use parallel to boot the VMs in parallel in the same way as before, but making it easier to track the success or failure of individual scenarios. Add the --progress flag when stdout for this script is a terminal to aid developers working locally when debugging. --- test/bin/ci_phase_iso_boot.sh | 51 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/test/bin/ci_phase_iso_boot.sh b/test/bin/ci_phase_iso_boot.sh index 6f6d44071b..af05dd85ec 100755 --- a/test/bin/ci_phase_iso_boot.sh +++ b/test/bin/ci_phase_iso_boot.sh @@ -19,6 +19,8 @@ echo "Logging to ${LOGFILE}" # Set fd 1 and 2 to write to the log file exec &> >(tee >(awk '{ print strftime("%Y-%m-%d %H:%M:%S"), $0; fflush() }' >"${LOGFILE}")) +LAUNCH_VMS_JOB_LOG="${IMAGEDIR}/launch_vm_jobs.txt" + cd "${ROOTDIR}" # Make sure libvirtd is running. We do this here, because some of the @@ -37,31 +39,28 @@ bash -x ./bin/manage_hypervisor_config.sh create # repository. bash -x ./bin/start_webserver.sh -declare -A pidToScenario - -# Build all of the needed VMs -for scenario in "${SCENARIO_SOURCES}"/*.sh; do - scenario_name=$(basename "${scenario}" .sh) - logfile="${SCENARIO_INFO_DIR}/${scenario_name}/boot.log" - mkdir -p "$(dirname "${logfile}")" - bash -x ./bin/scenario.sh create "${scenario}" >"${logfile}" 2>&1 & - pidToScenario["$!"]="${scenario}" - sleep 5 -done - -set +x -for pid in "${!pidToScenario[@]}"; do echo "${pid} - ${pidToScenario[${pid}]}"; done -set -x - -FAIL=0 -for job in $(jobs -p); do - jobs -l - echo "Waiting for job: ${job}" - if ! wait "${job}"; then - ((FAIL += 1)) - echo "Failed to boot VMs for scenario: ${pidToScenario[${job}]}" - fi -done +# Show the summary of the output of the parallel jobs. +if [ -t 0 ]; then + progress="--progress" +else + progress="" +fi + +# Tell scenario.sh to merge stderr into stdout +export SCENARIO_MERGE_OUTPUT_STREAMS=true + +LAUNCH_OK=true +if ! parallel \ + ${progress} \ + --results "${SCENARIO_INFO_DIR}/{/.}/boot.log" \ + --joblog "${LAUNCH_VMS_JOB_LOG}" \ + --delay 5 \ + bash -x ./bin/scenario.sh create ::: "${SCENARIO_SOURCES}"/*.sh ; then + LAUNCH_OK=false +fi + +# Show the summary of the output of the parallel jobs. +cat "${LAUNCH_VMS_JOB_LOG}" echo "====================================" echo "System information after booting VMs" @@ -72,7 +71,7 @@ sudo du -sk "${IMAGEDIR}"/* | sort -n sudo virsh list --all echo "====================================" -if [ ${FAIL} -ne 0 ]; then +if ! "${LAUNCH_OK}"; then echo "Failed to boot all VMs" exit 1 fi From 525e79fb9aabb5b10fede953d790e8f4892bb8b8 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Tue, 28 Nov 2023 18:20:21 -0500 Subject: [PATCH 5/8] use parallel to run scenario tests Use parallel to run the test scenarios in parallel in the same way as before, but making it easier to track the success or failure of individual scenarios. Add the --progress flag when stdout for this script is a terminal to aid developers working locally when debugging. --- test/bin/ci_phase_test.sh | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/test/bin/ci_phase_test.sh b/test/bin/ci_phase_test.sh index e8bcad5d32..6f1ce78748 100755 --- a/test/bin/ci_phase_test.sh +++ b/test/bin/ci_phase_test.sh @@ -9,27 +9,36 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # shellcheck source=test/bin/common.sh source "${SCRIPTDIR}/common.sh" +TEST_JOB_LOG="${IMAGEDIR}/test_jobs.txt" + cd "${TESTDIR}" if [ ! -d "${RF_VENV}" ]; then "${ROOTDIR}/scripts/fetch_tools.sh" robotframework fi -for scenario in "${SCENARIO_SOURCES}"/*.sh; do - scenario_name="$(basename "${scenario}" .sh)" - logfile="${SCENARIO_INFO_DIR}/${scenario_name}/run.log" - mkdir -p "$(dirname "${logfile}")" - SSH_PRIVATE_KEY="${HOME}/.ssh/id_rsa" bash -x ./bin/scenario.sh run "${scenario}" >"${logfile}" 2>&1 & -done +# Tell scenario.sh to merge stderr into stdout +export SCENARIO_MERGE_OUTPUT_STREAMS=true + +# Show the summary of the output of the parallel jobs. +if [ -t 0 ]; then + progress="--progress" +else + progress="" +fi + +TEST_OK=true +if ! parallel \ + ${progress} \ + --results "${SCENARIO_INFO_DIR}/{/.}/run.log" \ + --joblog "${TEST_JOB_LOG}" \ + bash -x ./bin/scenario.sh run ::: "${SCENARIO_SOURCES}"/*.sh ; then + TEST_OK=false +fi -FAIL=0 -for job in $(jobs -p) ; do - jobs -l - echo "Waiting for job: ${job}" - wait "${job}" || ((FAIL+=1)) -done +cat "${TEST_JOB_LOG}" echo "Test phase complete" -if [[ ${FAIL} -ne 0 ]]; then +if ! "${TEST_OK}"; then exit 1 fi From c15c936ff4ba8657e6a6cd6fef8f1ed418492532 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 29 Nov 2023 08:40:14 -0500 Subject: [PATCH 6/8] use explicit exit 0 in ci phase scripts Use an explicit exit when ending the script without any errors to avoid any potential doubt about what the return code from the script should be. --- test/bin/ci_phase_iso_boot.sh | 1 + test/bin/ci_phase_iso_build.sh | 1 + test/bin/ci_phase_test.sh | 2 ++ 3 files changed, 4 insertions(+) diff --git a/test/bin/ci_phase_iso_boot.sh b/test/bin/ci_phase_iso_boot.sh index af05dd85ec..3269f3147f 100755 --- a/test/bin/ci_phase_iso_boot.sh +++ b/test/bin/ci_phase_iso_boot.sh @@ -77,3 +77,4 @@ if ! "${LAUNCH_OK}"; then fi echo "Boot phase complete" +exit 0 diff --git a/test/bin/ci_phase_iso_build.sh b/test/bin/ci_phase_iso_build.sh index bd2b03b576..3bd29c1f8e 100755 --- a/test/bin/ci_phase_iso_build.sh +++ b/test/bin/ci_phase_iso_build.sh @@ -160,3 +160,4 @@ else fi echo "Build phase complete" +exit 0 diff --git a/test/bin/ci_phase_test.sh b/test/bin/ci_phase_test.sh index 6f1ce78748..0eb02a532b 100755 --- a/test/bin/ci_phase_test.sh +++ b/test/bin/ci_phase_test.sh @@ -40,5 +40,7 @@ cat "${TEST_JOB_LOG}" echo "Test phase complete" if ! "${TEST_OK}"; then + echo "Some tests failed" exit 1 fi +exit 0 From 735db7d9c5fbb086cb5e73f503e0bcc41f33ef7e Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 29 Nov 2023 13:19:59 -0500 Subject: [PATCH 7/8] add junit failure messages for error cases in run_tests If any step fails before the tests are run, we should report that failure in a way that makes the job failure easier to debug. --- test/bin/scenario.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 007d2318a0..f7e7c7d663 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -574,13 +574,17 @@ run_tests() { if [ ! -d "${RF_VENV}" ]; then error "RF_VENV (${RF_VENV}) does not exist, create it with: ${ROOTDIR}/scripts/fetch_tools.sh robotframework" + record_junit "${vmname}" "robot_framework_environment" "FAILED" exit 1 fi + record_junit "${vmname}" "robot_framework_environment" "OK" local rf_binary="${RF_VENV}/bin/robot" if [ ! -f "${rf_binary}" ]; then error "robot is not installed to ${rf_binary}" + record_junit "${vmname}" "robot_framework_installed" "FAILED" exit 1 fi + record_junit "${vmname}" "robot_framework_installed" "OK" # The IP file is created empty during the launch VM phase if the VM is has no NICs. This is the queue to skip # the variable file creation and greenboot check. @@ -595,8 +599,10 @@ run_tests() { f="$(vm_property_filename "${vmname}" "${p}")" if [ ! -f "${f}" ]; then error "Cannot read ${f}" + record_junit "${vmname}" "access_vm_property ${p}" "FAILED" exit 1 fi + record_junit "${vmname}" "access_vm_property ${p}" "OK" done local -r ssh_port=$(get_vm_property "${vmname}" "ssh_port") local -r api_port=$(get_vm_property "${vmname}" "api_port") @@ -617,8 +623,10 @@ SSH_PRIV_KEY: "${SSH_PRIVATE_KEY:-}" SSH_PORT: ${ssh_port} EOF if ! wait_for_greenboot "${full_vmname}" "${vm_ip}"; then + record_junit "${vmname}" "pre_test_greenboot_check" "FAILED" return 1 fi + record_junit "${vmname}" "pre_test_greenboot_check" "OK" fi local var_arg=${variable_file:+-V "${variable_file}"} From 474ac7be45a7bcd9c248ec2cb79b930dc0c6127d Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 29 Nov 2023 15:07:25 -0500 Subject: [PATCH 8/8] restructure junit in scenario.sh We need to collect junit output for multiple iterations of calling scenario.sh for the same job. We could use different output files, but that would mean complicating when we close the XML data structure. Using separate files is simpler. The output file has to be called 'junit.xml', so we want to put the files in separate directories. This commit adds a "phase" directory for each scenario to hold the separate junit files. --- test/bin/scenario.sh | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index f7e7c7d663..89db47dd80 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -274,21 +274,18 @@ wait_for_greenboot() { } start_junit() { - local outputfile="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/junit.xml" - mkdir -p "$(dirname "${outputfile}")" + mkdir -p "$(dirname "${JUNIT_OUTPUT_FILE}")" - echo "Creating ${outputfile}" + echo "Creating ${JUNIT_OUTPUT_FILE}" - cat - >"${outputfile}" <"${JUNIT_OUTPUT_FILE}" < EOF } close_junit() { - local outputfile="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/junit.xml" - - echo '' >>"${outputfile}" + echo '' >>"${JUNIT_OUTPUT_FILE}" } record_junit() { @@ -296,9 +293,7 @@ record_junit() { local step="$2" local results="$3" - local outputfile="${SCENARIO_INFO_DIR}/${SCENARIO}/vms/junit.xml" - - cat - >>"${outputfile}" <>"${JUNIT_OUTPUT_FILE}" < EOF @@ -306,17 +301,17 @@ EOF OK) ;; SKIP*) - cat - >>"${outputfile}" <>"${JUNIT_OUTPUT_FILE}" < EOF ;; *) - cat - >>"${outputfile}" <>"${JUNIT_OUTPUT_FILE}" < EOF esac - cat - >>"${outputfile}" <>"${JUNIT_OUTPUT_FILE}" < EOF } @@ -570,6 +565,9 @@ run_tests() { local -r full_vmname="$(full_vm_name "${vmname}")" shift + start_junit + trap "close_junit" EXIT + echo "Running tests with $# args" "$@" if [ ! -d "${RF_VENV}" ]; then @@ -760,6 +758,7 @@ shift SCENARIO_SCRIPT="$(realpath "$1")" shift SCENARIO=$(basename "${SCENARIO_SCRIPT}" .sh) +JUNIT_OUTPUT_FILE="${SCENARIO_INFO_DIR}/${SCENARIO}/phase_${action}/junit.xml" # Change directory to the test root cd "${SCRIPTDIR}/.."