From 7c5517d5a96f37cbf42744f842b7051ad6eb2842 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 24 Sep 2020 21:55:45 -0700 Subject: [PATCH] ci-operator/step-registry/openshift/e2e/aws/proxy: Skip failing tests A recent proxy job failed the following test-cases [1]: [sig-arch] Managed cluster should should expose cluster services outside the cluster [Suite:openshift/conformance/parallel] [sig-builds][Feature:Builds] build have source revision metadata started build should contain source revision information [Suite:openshift/conformance/parallel] [sig-cli] oc adm must-gather runs successfully for audit logs [Suite:openshift/conformance/parallel] [sig-cluster-lifecycle][Feature:Machines] Managed cluster should have machine resources [Suite:openshift/conformance/parallel] [sig-imageregistry][Feature:ImageAppend] Image append should create images by appending them [Suite:openshift/conformance/parallel] [sig-imageregistry][Feature:ImageInfo] Image info should display information about images [Suite:openshift/conformance/parallel] [sig-network] Internal connectivity for TCP and UDP on ports 9000-9999 is allowed [Suite:openshift/conformance/parallel] [sig-network] Networking should provide Internet connection for containers [Feature:Networking-IPv4] [Skipped:azure] [Suite:openshift/conformance/parallel] [Suite:k8s] [sig-network][Feature:Router] The HAProxy router should respond with 503 to unrecognized hosts [Suite:openshift/conformance/parallel] [sig-network][Feature:Router] The HAProxy router should serve routes that were created from an ingress [Suite:openshift/conformance/parallel] [sig-network][Feature:Router] The HAProxy router should set Forwarded headers appropriately [Suite:openshift/conformance/parallel] [sig-network][Feature:Router] The HAProxy router should support reencrypt to services backed by a serving certificate automatically [Suite:openshift/conformance/parallel] This commit adds a TEST_SKIPS framework, taking advantage of: $ openshift-tests run --help | grep 'dry-run\|--file' If you specify the --dry-run argument, the names of each individual test that is part of the suite will be printed, one per line. You may filter this list and pass it back to the run command with the --file argument. You may also pipe a list of test names, one per line, on standard input by passing "-f -". --dry-run Print the tests to run without executing them. -f, --file string Create a suite from the newline-delimited test names in this file. Grep uses basic regular expressions by default [2]. The YAML `>-` is trimmed line folding [3], so we get one long line with no trailing newline: $ yaml2json - + Image append should create images by appending them\| + Image info should display information about images\| + Internal connectivity for TCP and UDP on ports 9000-9999 is allowed\| + Managed cluster should have machine resources\| + Managed cluster should should expose cluster services outside the cluster\| + Networking should provide Internet connection for containers\| + The HAProxy router should respond with 503 to unrecognized hosts\| + The HAProxy router should serve routes that were created from an ingress\| + The HAProxy router should set Forwarded headers appropriately\| + The HAProxy router should support reencrypt to services backed by a serving certificate automatically\| + build have source revision metadata started build should contain source revision information\| + oc adm must-gather runs successfully for audit logs documentation: |- - The Openshift E2E AWS workflow executes the common end-to-end test suite on AWS with a proxy cluster configuration. \ No newline at end of file + The Openshift E2E AWS workflow executes the common end-to-end test suite on AWS with a proxy cluster configuration. diff --git a/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh b/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh index b3408a0559e34..8e79e7167c5d9 100644 --- a/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh +++ b/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-commands.sh @@ -67,12 +67,20 @@ if [[ "${CLUSTER_TYPE}" == gcp ]]; then fi if [[ -n "${TEST_OPTIONS}" ]]; then - export TEST_ARGS="--options=${TEST_OPTIONS}" + TEST_ARGS="--options=${TEST_OPTIONS}" fi if [[ "${TEST_COMMAND}" == "run-upgrade" && -n "${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}" ]]; then # OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE is a pullspec of release:latest imagestreamtag - export TEST_ARGS="${TEST_ARGS:-} --to-image=${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}" + TEST_ARGS="${TEST_ARGS:-} --to-image=${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}" +fi + +if [[ -n "${TEST_SKIPS}" ]]; then + TESTS="$(openshift-tests "${TEST_COMMAND}" --dry-run "${TEST_SUITE}")" + echo "${TESTS}" | grep -v "${TEST_SKIPS}" >/tmp/tests + echo "Skipping tests:" + echo "${TESTS}" | grep "${TEST_SKIPS}" + TEST_ARGS="${TEST_ARGS:-} --file /tmp/tests" fi openshift-tests "${TEST_COMMAND}" "${TEST_SUITE}" ${TEST_ARGS:-} \ diff --git a/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-ref.yaml b/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-ref.yaml index 50244f0e88456..0e511454c1c02 100644 --- a/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-ref.yaml +++ b/ci-operator/step-registry/openshift/e2e/test/openshift-e2e-test-ref.yaml @@ -9,6 +9,9 @@ ref: - name: TEST_SUITE default: openshift/conformance/parallel documentation: The test suite to run. Use 'openshift-test TEST_COMMAND --help' to list available suites. + - name: TEST_SKIPS + default: "" + documentation: Regular expression (POSIX basic regular expression) of tests to skip. - name: TEST_OPTIONS default: "" documentation: The test command options. Use 'openshift-test TEST_COMMAND --help' to list available options.