From 8367e9fca50492ba3b9d0a81afab15b42705866f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gull=C3=B3n?= Date: Tue, 4 Feb 2025 16:49:52 +0100 Subject: [PATCH 1/7] poc to decouple rf test execution from scenario.sh script --- test/bin/run_rf_tests.sh | 149 +++++++++++++++++++++++++++++++++++++++ test/bin/scenario.sh | 29 +++----- test/run.sh | 130 ---------------------------------- 3 files changed, 157 insertions(+), 151 deletions(-) create mode 100755 test/bin/run_rf_tests.sh delete mode 100755 test/run.sh diff --git a/test/bin/run_rf_tests.sh b/test/bin/run_rf_tests.sh new file mode 100755 index 0000000000..eedb405b1f --- /dev/null +++ b/test/bin/run_rf_tests.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +set -xeuo pipefail +IFS=$'\n\t' + +# path vars +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="${SCRIPTDIR}/" + +# default config +DRYRUN="" +OUTDIR="${ROOTDIR}/_output/e2e-$(date +%Y%m%d-%H%M%S)" +RF_VENV="${ROOTDIR}/_output/robotenv" +RF_VARIABLES="${SCRIPTDIR}/variables.yaml" +SCENARIO="Default Name" +TEST_RANDOMIZATION="all" +TEST_EXECUTION_TIMEOUT="300m" +SCENARIO="Default Scenario" +EXITONFAILURE="" + +function usage { + local -r script_name=$(basename "$0") + cat - <&2 + echo "See ${SCRIPTDIR}/variables.yaml.example for the expected content." 1>&2 + exit 1 +fi + +cd "${SCRIPTDIR}" || (echo "Did not find ${SCRIPTDIR}" 1>&2; exit 1) + +TESTS="$*" +if [ -z "${TESTS}" ]; then + echo "ERROR: missing which RF suites/tests to run" + exit 1 +fi + +# enable stress condition +if [ "${STRESS_TESTING:-}" ]; then + # DEST_DIR var is the python env dir used by fetch_tools.sh to install the tools + export DEST_DIR="${RF_VENV}" + "${ROOTDIR}/scripts/fetch_tools.sh" yq + YQ_BINARY="${RF_VENV}/yq" + + CONDITION="${STRESS_TESTING%=*}" + VALUE="${STRESS_TESTING#*=}" + + SSH_HOST=$("${YQ_BINARY}" '.USHIFT_HOST' "${RF_VARIABLES}") + SSH_USER=$("${YQ_BINARY}" '.USHIFT_USER' "${RF_VARIABLES}") + SSH_PORT=$("${YQ_BINARY}" '.SSH_PORT' "${RF_VARIABLES}") + SSH_PKEY=$("${YQ_BINARY}" '.SSH_PRIV_KEY' "${RF_VARIABLES}") + + "${SCRIPTDIR}"/bin/stress_testing.sh -e "${CONDITION}" -v "${VALUE}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" +fi + +# Make sure the test execution times out after a predefined period. +# The 'timeout' command sends the HUP signal and, if the test does not +# exit after 5m, it sends the KILL signal to terminate the process. +timeout_robot="timeout -v --kill-after=5m ${TEST_EXECUTION_TIMEOUT} ${RF_BINARY}" +if [ -t 0 ]; then + # Disable timeout for interactive mode when stdin is a terminal. + # This is necessary for proper handling of test interruption by user. + timeout_robot="${RF_BINARY}" +fi + +# shellcheck disable=SC2086,SC2068 +"${timeout_robot}" \ + ${DRYRUN} \ + ${EXITONFAILURE} \ + --name "${SCENARIO}" \ + --randomize "${TEST_RANDOMIZATION}" \ + --prerunmodifier "${SCRIPTDIR}/../resources/SkipTests.py:${SKIP_TESTS:-}" \ + --loglevel TRACE \ + --outputdir "${OUTDIR}" \ + --debugfile "${OUTDIR}/rf-debug.log" \ + -V "${RF_VARIABLES}" \ + -x junit.xml \ + ${TESTS[@]} + +# disable stress condition +if [ "${STRESS_TESTING:-}" ]; then + "${SCRIPTDIR}"/bin/stress_testing.sh -d "${CONDITION}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" +fi diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 18ebb44c9a..8c8ede5944 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -931,27 +931,14 @@ EOF record_junit "${vmname}" "pre_test_greenboot_check" "OK" fi - # Make sure the test execution times out after a predefined period. - # The 'timeout' command sends the HUP signal and, if the test does not - # exit after 5m, it sends the KILL signal to terminate the process. - local var_arg=${variable_file:+-V "${variable_file}"} - local timeout_robot="timeout -v --kill-after=5m ${TEST_EXECUTION_TIMEOUT} ${rf_binary}" - if [ -t 0 ]; then - # Disable timeout for interactive mode when stdin is a terminal. - # This is necessary for proper handling of test interruption by user. - timeout_robot="${rf_binary}" - fi - - # shellcheck disable=SC2086 - if ! ${timeout_robot} \ - --name "${SCENARIO}" \ - --randomize "${TEST_RANDOMIZATION}" \ - --loglevel TRACE \ - --outputdir "${SCENARIO_INFO_DIR}/${SCENARIO}" \ - --debugfile "${SCENARIO_INFO_DIR}/${SCENARIO}/rf-debug.log" \ - -x junit.xml \ - ${var_arg} \ - "$@" ; then + if ! "./${SCRIPTDIR}/run_rf_tests.sh" \ + -i "${variable_file}" \ + -b "${rf_binary}" \ + -o "${SCENARIO_INFO_DIR}/${SCENARIO}" \ + -s "${SCENARIO}" \ + -r "${TEST_RANDOMIZATION}" \ + -t "${TEST_EXECUTION_TIMEOUT}" \ + ; then # Log junit message on the command timeout if [ $? -ge 124 ] ; then record_junit "${vmname}" "run_test_timed_out_${TEST_EXECUTION_TIMEOUT}" "FAILED" diff --git a/test/run.sh b/test/run.sh deleted file mode 100755 index 19161f0ab6..0000000000 --- a/test/run.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash - -set -xeuo pipefail -IFS=$'\n\t' - -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOTDIR="${SCRIPTDIR}/.." - -RF_VENV="${ROOTDIR}/_output/robotenv" -RF_VARIABLES="${SCRIPTDIR}/variables.yaml" -DRYRUN=false -OUTDIR="${ROOTDIR}/_output/e2e-$(date +%Y%m%d-%H%M%S)" -RANDOMIZE=all -EXITONFAILURE="" - -function usage { - local -r script_name=$(basename "$0") - cat - <&2 - echo "See ${SCRIPTDIR}/variables.yaml.example for the expected content." 1>&2 - exit 1 -fi - -# DEST_DIR var is the python env dir used by fetch_tools.sh to install the tools -export DEST_DIR="${RF_VENV}" -"${ROOTDIR}/scripts/fetch_tools.sh" robotframework -"${ROOTDIR}/scripts/fetch_tools.sh" yq - -RF_BINARY="${RF_VENV}/bin/robot" -YQ_BINARY="${RF_VENV}/yq" - -cd "${SCRIPTDIR}" || (echo "Did not find ${SCRIPTDIR}" 1>&2; exit 1) - -TESTS="$*" -# if TESTS is not set - run the standard suite. -if [ -z "${TESTS}" ]; then - TESTS=(./suites/standard1 ./suites/standard2 ./suites/osconfig/clusterid.robot ./suites/storage/storage-version-migration.robot) -fi - -# enable stress condition -if [ "${STRESS_TESTING:-}" ]; then - CONDITION="${STRESS_TESTING%=*}" - VALUE="${STRESS_TESTING#*=}" - - SSH_HOST=$("${YQ_BINARY}" '.USHIFT_HOST' "${RF_VARIABLES}") - SSH_USER=$("${YQ_BINARY}" '.USHIFT_USER' "${RF_VARIABLES}") - SSH_PORT=$("${YQ_BINARY}" '.SSH_PORT' "${RF_VARIABLES}") - SSH_PKEY=$("${YQ_BINARY}" '.SSH_PRIV_KEY' "${RF_VARIABLES}") - - "${SCRIPTDIR}"/bin/stress_testing.sh -e "${CONDITION}" -v "${VALUE}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" -fi - -if ${DRYRUN}; then - # shellcheck disable=SC2086,SC2068 - "${RF_BINARY}" \ - --dryrun \ - --outputdir "${OUTDIR}" \ - ${TESTS[@]} -else - # shellcheck disable=SC2086,SC2068 - "${RF_BINARY}" \ - ${EXITONFAILURE} \ - --randomize "${RANDOMIZE}" \ - --prerunmodifier "${SCRIPTDIR}/resources/SkipTests.py:${SKIP_TESTS:-}" \ - --loglevel TRACE \ - -V "${RF_VARIABLES}" \ - -x junit.xml \ - --outputdir "${OUTDIR}" \ - ${TESTS[@]} -fi - -# disable stress condition -if [ "${STRESS_TESTING:-}" ]; then - "${SCRIPTDIR}"/bin/stress_testing.sh -d "${CONDITION}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" -fi From 540f96f6c09933f66f2538d3bd7b3a5b76c6c3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gull=C3=B3n?= Date: Wed, 5 Feb 2025 10:05:45 +0100 Subject: [PATCH 2/7] fix path --- test/bin/scenario.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index a5f22df1e0..3c522b6559 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -933,7 +933,7 @@ EOF record_junit "${vmname}" "pre_test_greenboot_check" "OK" fi - if ! "./${SCRIPTDIR}/run_rf_tests.sh" \ + if ! "${SCRIPTDIR}/run_rf_tests.sh" \ -i "${variable_file}" \ -b "${rf_binary}" \ -o "${SCENARIO_INFO_DIR}/${SCENARIO}" \ From def1804c83526d0589d219faa8adadb0f1a3ae5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gull=C3=B3n?= Date: Wed, 5 Feb 2025 13:08:04 +0100 Subject: [PATCH 3/7] fix path to tests to run --- test/bin/scenario.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 3c522b6559..c7aae0c527 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -940,7 +940,8 @@ EOF -s "${SCENARIO}" \ -r "${TEST_RANDOMIZATION}" \ -t "${TEST_EXECUTION_TIMEOUT}" \ - ; then + "$@" ; \ + then # Log junit message on the command timeout if [ $? -ge 124 ] ; then record_junit "${vmname}" "run_test_timed_out_${TEST_EXECUTION_TIMEOUT}" "FAILED" From 2ddd60e356b223471aa105b4941fdd44603827e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gull=C3=B3n?= Date: Wed, 5 Feb 2025 17:26:16 +0100 Subject: [PATCH 4/7] fix paths --- test/bin/run_rf_tests.sh | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/test/bin/run_rf_tests.sh b/test/bin/run_rf_tests.sh index eedb405b1f..9bfc0fa480 100755 --- a/test/bin/run_rf_tests.sh +++ b/test/bin/run_rf_tests.sh @@ -4,14 +4,13 @@ set -xeuo pipefail IFS=$'\n\t' # path vars -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ROOTDIR="${SCRIPTDIR}/" +TEST_BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # default config DRYRUN="" -OUTDIR="${ROOTDIR}/_output/e2e-$(date +%Y%m%d-%H%M%S)" -RF_VENV="${ROOTDIR}/_output/robotenv" -RF_VARIABLES="${SCRIPTDIR}/variables.yaml" +OUTDIR="${TEST_BIN_DIR}/_output/e2e-$(date +%Y%m%d-%H%M%S)" +RF_VENV="${TEST_BIN_DIR}/_output/robotenv" +RF_VARIABLES="${TEST_BIN_DIR}/variables.yaml" SCENARIO="Default Name" TEST_RANDOMIZATION="all" TEST_EXECUTION_TIMEOUT="300m" @@ -89,11 +88,11 @@ shift $((OPTIND-1)) if [ ! -f "${RF_VARIABLES}" ]; then echo "Please create or provide a variables file at ${RF_VARIABLES}" 1>&2 - echo "See ${SCRIPTDIR}/variables.yaml.example for the expected content." 1>&2 + echo "See ${TEST_BIN_DIR}/variables.yaml.example for the expected content." 1>&2 exit 1 fi -cd "${SCRIPTDIR}" || (echo "Did not find ${SCRIPTDIR}" 1>&2; exit 1) +cd "${TEST_BIN_DIR}" || (echo "Did not find ${TEST_BIN_DIR}" 1>&2; exit 1) TESTS="$*" if [ -z "${TESTS}" ]; then @@ -105,7 +104,7 @@ fi if [ "${STRESS_TESTING:-}" ]; then # DEST_DIR var is the python env dir used by fetch_tools.sh to install the tools export DEST_DIR="${RF_VENV}" - "${ROOTDIR}/scripts/fetch_tools.sh" yq + "${TEST_BIN_DIR}/../../scripts/fetch_tools.sh" yq YQ_BINARY="${RF_VENV}/yq" CONDITION="${STRESS_TESTING%=*}" @@ -116,7 +115,7 @@ if [ "${STRESS_TESTING:-}" ]; then SSH_PORT=$("${YQ_BINARY}" '.SSH_PORT' "${RF_VARIABLES}") SSH_PKEY=$("${YQ_BINARY}" '.SSH_PRIV_KEY' "${RF_VARIABLES}") - "${SCRIPTDIR}"/bin/stress_testing.sh -e "${CONDITION}" -v "${VALUE}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" + "${TEST_BIN_DIR}/stress_testing.sh" -e "${CONDITION}" -v "${VALUE}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" fi # Make sure the test execution times out after a predefined period. @@ -135,7 +134,7 @@ fi ${EXITONFAILURE} \ --name "${SCENARIO}" \ --randomize "${TEST_RANDOMIZATION}" \ - --prerunmodifier "${SCRIPTDIR}/../resources/SkipTests.py:${SKIP_TESTS:-}" \ + --prerunmodifier "${TEST_BIN_DIR}/../resources/SkipTests.py:${SKIP_TESTS:-}" \ --loglevel TRACE \ --outputdir "${OUTDIR}" \ --debugfile "${OUTDIR}/rf-debug.log" \ @@ -145,5 +144,5 @@ fi # disable stress condition if [ "${STRESS_TESTING:-}" ]; then - "${SCRIPTDIR}"/bin/stress_testing.sh -d "${CONDITION}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" + "${TEST_BIN_DIR}/stress_testing.sh" -d "${CONDITION}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" fi From 95b3b39c9710edc62115f1b343e742041ce6aafe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gull=C3=B3n?= Date: Thu, 6 Feb 2025 09:43:45 +0100 Subject: [PATCH 5/7] fix scenario param --- test/bin/scenario.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index c7aae0c527..7ef151de0b 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -937,7 +937,7 @@ EOF -i "${variable_file}" \ -b "${rf_binary}" \ -o "${SCENARIO_INFO_DIR}/${SCENARIO}" \ - -s "${SCENARIO}" \ + -c "${SCENARIO}" \ -r "${TEST_RANDOMIZATION}" \ -t "${TEST_EXECUTION_TIMEOUT}" \ "$@" ; \ From 140267e6f478f92dbd2f3082dbf59a14e154527d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gull=C3=B3n?= Date: Thu, 6 Feb 2025 11:10:22 +0100 Subject: [PATCH 6/7] fix path --- test/bin/run_rf_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/bin/run_rf_tests.sh b/test/bin/run_rf_tests.sh index 9bfc0fa480..d9af979075 100755 --- a/test/bin/run_rf_tests.sh +++ b/test/bin/run_rf_tests.sh @@ -129,7 +129,7 @@ if [ -t 0 ]; then fi # shellcheck disable=SC2086,SC2068 -"${timeout_robot}" \ +${timeout_robot} \ ${DRYRUN} \ ${EXITONFAILURE} \ --name "${SCENARIO}" \ From f055aa960df3930d4db6f35282dda7087da2f679 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Gull=C3=B3n?= Date: Thu, 6 Feb 2025 12:24:27 +0100 Subject: [PATCH 7/7] move out timeout rf_binary --- test/bin/run_rf_tests.sh | 12 +----------- test/bin/scenario.sh | 12 +++++++++++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/bin/run_rf_tests.sh b/test/bin/run_rf_tests.sh index d9af979075..f894e3394f 100755 --- a/test/bin/run_rf_tests.sh +++ b/test/bin/run_rf_tests.sh @@ -118,18 +118,8 @@ if [ "${STRESS_TESTING:-}" ]; then "${TEST_BIN_DIR}/stress_testing.sh" -e "${CONDITION}" -v "${VALUE}" -h "${SSH_HOST}" -u "${SSH_USER}" -p "${SSH_PORT}" -k "${SSH_PKEY}" fi -# Make sure the test execution times out after a predefined period. -# The 'timeout' command sends the HUP signal and, if the test does not -# exit after 5m, it sends the KILL signal to terminate the process. -timeout_robot="timeout -v --kill-after=5m ${TEST_EXECUTION_TIMEOUT} ${RF_BINARY}" -if [ -t 0 ]; then - # Disable timeout for interactive mode when stdin is a terminal. - # This is necessary for proper handling of test interruption by user. - timeout_robot="${RF_BINARY}" -fi - # shellcheck disable=SC2086,SC2068 -${timeout_robot} \ +${RF_BINARY} \ ${DRYRUN} \ ${EXITONFAILURE} \ --name "${SCENARIO}" \ diff --git a/test/bin/scenario.sh b/test/bin/scenario.sh index 7ef151de0b..3b364fe81e 100755 --- a/test/bin/scenario.sh +++ b/test/bin/scenario.sh @@ -933,9 +933,19 @@ EOF record_junit "${vmname}" "pre_test_greenboot_check" "OK" fi + # Make sure the test execution times out after a predefined period. + # The 'timeout' command sends the HUP signal and, if the test does not + # exit after 5m, it sends the KILL signal to terminate the process. + timeout_robot="timeout -v --kill-after=5m ${TEST_EXECUTION_TIMEOUT} ${rf_binary}" + if [ -t 0 ]; then + # Disable timeout for interactive mode when stdin is a terminal. + # This is necessary for proper handling of test interruption by user. + timeout_robot="${rf_binary}" + fi + if ! "${SCRIPTDIR}/run_rf_tests.sh" \ -i "${variable_file}" \ - -b "${rf_binary}" \ + -b "${timeout_robot}" \ -o "${SCENARIO_INFO_DIR}/${SCENARIO}" \ -c "${SCENARIO}" \ -r "${TEST_RANDOMIZATION}" \