USHIFT-6175: Ginkgo releases tests are failing in CI

scripts/fetch_tools.sh

@@ -260,60 +260,62 @@ gettool_ginkgo() {
     source "${SCRIPT_DIR}/../test/bin/common_versions.sh"
 
     local -r repo_url="https://${GITHUB_TOKEN}@github.com/openshift/openshift-tests-private.git"
+    local -r repo_branch="${OPENSHIFT_TESTS_PRIVATE_REPO_BRANCH}"
+    local -r repo_commit="${OPENSHIFT_TESTS_PRIVATE_REPO_COMMIT}"
+    local clone_dir="${WORK_DIR}/openshift-tests-private"
+
     local -r binary_path="${GINKGO_TEST_BINARY}"
-    local -r release_branch="release-4.${MINOR_VERSION}"
-
-    # Skip if binary already exists
-    [[ -f "${binary_path}" ]] && return 0
+    local -r handleresult_script="${HANDLERESULT_SCRIPT}"
 
     # Check if Go is available
     if ! command -v go &> /dev/null; then
         echo "Error: Go is required to build openshift-tests-private binary"
         return 1
     fi
 
-    # Ensure binary directory exists
+    # Check if binary and handleresult.py already exist
+    [[ -f "${binary_path}" ]] && [[ -f "${handleresult_script}" ]] && return 0
     mkdir -p "$(dirname "${binary_path}")"
 
-    # Use global WORK_DIR for cloning
-    local clone_dir="${WORK_DIR}/openshift-tests-private"
-
     # Clone repository with release branch preference
-    if ! git clone --depth 1 --branch "${release_branch}" "${repo_url}" "${clone_dir}" 2>/dev/null; then
-        echo "Branch ${release_branch} not found, cloning default branch..."
-        if ! git clone --depth 1 "${repo_url}" "${clone_dir}"; then
-            echo "Error: Failed to clone repository"
-            return 1
-        fi
+    if ! git clone --depth 1000 --branch "${repo_branch}" "${repo_url}" "${clone_dir}" 2>/dev/null; then
+        echo "Error: Failed to clone repository"
+        return 1
     fi
 
-    # Build the binary
     pushd "${clone_dir}" &>/dev/null
 
-    local test_binary="./bin/extended-platform-tests"
-    make build
+    # Checkout a valid commit to be sure ginkgo tests are working
+    if ! (git checkout --quiet "${repo_commit}"); then
+        echo "Error: Failed to checkout specific commit"
+        return 1
+    fi
+
+    # Build the binary
+    make all
 
     # Copy binary to centralized location
+    local test_binary="./bin/extended-platform-tests"
     if [[ -f "${test_binary}" ]]; then
         cp "${test_binary}" "${binary_path}"
         chmod +x "${binary_path}"
         echo "Binary installed to ${binary_path}"
-
-	    # Copy handleresult.py to the tools directory
-        if [[ -f "${clone_dir}/pipeline/handleresult.py" ]] && [[ -f "${HANDLERESULT_SCRIPT}" ]]; then
-            cp "${clone_dir}/pipeline/handleresult.py" "${HANDLERESULT_SCRIPT}"
-            chmod +x "${HANDLERESULT_SCRIPT}"
-            echo "handleresult.py installed to ${HANDLERESULT_SCRIPT}"
-        else
-            echo "Warning: pipeline/handleresult.py not found in repository"
-        fi
-
-        popd &>/dev/null
     else
         echo "Error: Test binary not found after build"
-        popd &>/dev/null
         return 1
     fi
+
+    # Copy handleresult.py to the tools directory
+    local handleresult_script_local="./pipeline/handleresult.py"
+    if [[ -f "${handleresult_script_local}" ]] && [[ ! -f "${handleresult_script}" ]]; then
+        cp "${handleresult_script_local}" "${handleresult_script}"
+        chmod +x "${handleresult_script}"
+        echo "handleresult.py installed to ${handleresult_script}"
+    else
+        echo "Error: pipeline/handleresult.py not found in repository"
+        return 1
+    fi
+    popd &>/dev/null
 }
 
 tool_getters=$(declare -F | awk '$3 ~ /^gettool_/ {print $3}' | sed 's/^gettool_//g')

test/bin/common_versions.sh

@@ -144,3 +144,9 @@ export LATEST_RELEASE_TYPE
 
 BREW_LREL_RELEASE_VERSION="${BREW_EC_RELEASE_VERSION}"
 export BREW_LREL_RELEASE_VERSION
+
+# Branch and commit for the openshift-tests-private repository
+OPENSHIFT_TESTS_PRIVATE_REPO_BRANCH="release-4.${MINOR_VERSION}"
+OPENSHIFT_TESTS_PRIVATE_REPO_COMMIT="61613d96c91db7b2907c24dd257075d3f2201991"
+export OPENSHIFT_TESTS_PRIVATE_REPO_BRANCH
+export OPENSHIFT_TESTS_PRIVATE_REPO_COMMIT

test/bin/scenario.sh

@@ -1152,21 +1152,34 @@ run_gingko_tests() {
     cat "${case_selected}"
     echo "-----------------------------------------------------"
 
-    # Run the tests and capture output
-    if ! "${GINKGO_TEST_BINARY}" run --timeout 60m --junit-dir="${test_results_dir}" -f "${case_selected}" 2>&1 | tee "${test_results_dir}/test-output.log"; then
-        record_junit "${vmname}" "run_gingko_tests" "FAILED"
-        return 1
+    # Make sure the test execution times out after a predefined period.
+    # The 'timeout' command sends the HUP signal and, if the test does not
+    # exit after 5m, it sends the KILL signal to terminate the process.
+    local timeout_ginkgo="timeout -v --kill-after=5m ${TEST_EXECUTION_TIMEOUT} ${GINKGO_TEST_BINARY}"
+    if [ -t 0 ]; then
+        # Disable timeout for interactive mode when stdin is a terminal.
+        # This is necessary for proper handling of test interruption by user.
+        # shellcheck disable=SC2034
+        timeout_ginkgo="${GINKGO_TEST_BINARY}"
     fi
 
-    record_junit "${vmname}" "run_gingko_tests" "OK"
+    # Run the tests and capture output with 10m timeout for every test case
+    echo "Gingko test execution started..."
+    ginkgo_result_success=true
+    if ! eval '${timeout_ginkgo} run --timeout 10m --junit-dir=${test_results_dir} -f ${case_selected}' 2>&1 | tee "${test_results_dir}/test-output.log"; then
+        if [ $? -ge 124 ] ; then
+            record_junit "${vmname}" "run_test_timed_out_${TEST_EXECUTION_TIMEOUT}" "FAILED"
+        fi
+        ginkgo_result_success=false
+    fi
+    echo "Gingko test execution completed"
     popd &>/dev/null
 
     # Clean the JUnit XML files
-    echo "Cleaning JUnit XML files..."
+    echo "Cleaning JUnit XML files to remove 'Monitor cluster while tests execute' test case"
     cleanup_success=true
-
     if [[ ! -f "${HANDLERESULT_SCRIPT}" ]]; then
-        echo "Warning: pipeline/handleresult.py not found. Skipping XML cleanup."
+        echo "Warning: ${HANDLERESULT_SCRIPT} not found. Skipping XML cleanup."
     else
         for junit_file in "${test_results_dir}"/junit_e2e_*.xml; do
             if [[ -f "${junit_file}" ]]; then
@@ -1180,7 +1193,6 @@ run_gingko_tests() {
                 temp_file="${junit_file}.tmp"
                 if python3 "${HANDLERESULT_SCRIPT}" -a replace -i "${junit_file}" -o "${temp_file}" 2>/dev/null; then
                     mv "${temp_file}" "${junit_file}"
-
                     echo "✓ Cleaned: ${filename}"
                     rm "${junit_file}.backup"  # Remove backup on success
                 else
@@ -1193,16 +1205,21 @@ run_gingko_tests() {
     fi
 
     # Display results summary
-    echo "Gingko test execution completed"
     echo "Results are available in: ${test_results_dir}"
     if [[ "${cleanup_success}" == "true" ]]; then
-      echo "✓ JUnit XML files have been cleaned (Monitor test cases removed)"
+        echo "Unit XML files have been cleaned ('Monitor cluster while tests execute' test case removed)"
+        record_junit "${vmname}" "clean_junit_xml_files" "OK"
     else
-      echo "⚠ Some XML files could not be cleaned (originals preserved)"
+        echo "Some XML files could not be cleaned (originals preserved)"
+        record_junit "${vmname}" "clean_junit_xml_files" "FAILED"
     fi
 
-    if [[ -f "${test_results_dir}/test-output.log" ]]; then
-        echo "Test output log: ${test_results_dir}/test-output.log"
+    # Record the junit result of the ginkgo tests
+    if [[ "${ginkgo_result_success}" == "true" ]]; then
+        record_junit "${vmname}" "run_gingko_tests" "OK"
+    else
+        record_junit "${vmname}" "run_gingko_tests" "FAILED"
+        exit 1
     fi
 }
 

test/scenarios-bootc/releases/el96-lrel@ginkgo-tests.sh

@@ -29,5 +29,5 @@ scenario_run_tests() {
         return 0
     fi
 
-    run_gingko_tests host1
+    run_gingko_tests host1 "~Disruptive"
 }

test/scenarios/releases/el96-lrel@backups.sh

@@ -5,7 +5,7 @@
 start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-${LATEST_RELEASE_TYPE}"
 
 scenario_create_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -15,7 +15,7 @@ scenario_create_vms() {
 }
 
 scenario_remove_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -24,7 +24,7 @@ scenario_remove_vms() {
 }
 
 scenario_run_tests() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi

test/scenarios/releases/el96-lrel@dual-stack.sh

@@ -5,7 +5,7 @@
 start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-${LATEST_RELEASE_TYPE}"
 
 scenario_create_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -15,7 +15,7 @@ scenario_create_vms() {
 }
 
 scenario_remove_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -24,7 +24,7 @@ scenario_remove_vms() {
 }
 
 scenario_run_tests() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi

test/scenarios/releases/el96-lrel@ginkgo-tests.sh

@@ -5,7 +5,7 @@
 start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-${LATEST_RELEASE_TYPE}"
 
 scenario_create_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -15,7 +15,7 @@ scenario_create_vms() {
 }
 
 scenario_remove_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -24,10 +24,10 @@ scenario_remove_vms() {
 }
 
 scenario_run_tests() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
 
-    run_gingko_tests host1
+    run_gingko_tests host1 "~Disruptive"
 }

test/scenarios/releases/el96-lrel@ipv6.sh

@@ -14,7 +14,7 @@ start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-${LATEST_RELE
 
 scenario_create_vms() {
     # Enable IPv6 single stack in kickstart
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -24,7 +24,7 @@ scenario_create_vms() {
 }
 
 scenario_remove_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -33,7 +33,7 @@ scenario_remove_vms() {
 }
 
 scenario_run_tests() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi

test/scenarios/releases/el96-lrel@low-latency.sh

@@ -8,7 +8,7 @@ export TEST_RANDOMIZATION=none
 start_image="rhel-9.6-microshift-brew-tuned-4.${MINOR_VERSION}-${LATEST_RELEASE_TYPE}"
 
 scenario_create_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -18,7 +18,7 @@ scenario_create_vms() {
 }
 
 scenario_remove_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -27,7 +27,7 @@ scenario_remove_vms() {
 }
 
 scenario_run_tests() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi

test/scenarios/releases/el96-lrel@multi-nic.sh

@@ -5,7 +5,7 @@
 start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-${LATEST_RELEASE_TYPE}"
 
 scenario_create_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -16,7 +16,7 @@ scenario_create_vms() {
 }
 
 scenario_remove_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -25,7 +25,7 @@ scenario_remove_vms() {
 }
 
 scenario_run_tests() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi

test/scenarios/releases/el96-lrel@optional.sh

@@ -10,7 +10,7 @@ WEB_SERVER_URL="http://${VM_BRIDGE_IP}:${WEB_SERVER_PORT}"
 start_image="rhel-9.6-microshift-brew-optionals-4.${MINOR_VERSION}-${LATEST_RELEASE_TYPE}"
 
 scenario_create_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -21,7 +21,7 @@ scenario_create_vms() {
 }
 
 scenario_remove_vms() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi
@@ -30,7 +30,7 @@ scenario_remove_vms() {
 }
 
 scenario_run_tests() {
-    if ! does_image_exist "${start_image}"; then
+    if ! does_commit_exist "${start_image}"; then
         echo "Image '${start_image}' not found - skipping test"
         return 0
     fi