From fb1a72f637998ed7a12a84a4f4e1dd24fe3c3713 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Thu, 10 Jun 2021 11:46:26 +0900 Subject: [PATCH 1/2] Run e2e test with net-istio enabled --- Makefile | 12 ++++++++-- hack/lib/serverless.bash | 45 ++++++++++++++++++++++++++++++++++++++ hack/lib/vars.bash | 1 + test/serving.bash | 18 ++++++++++++++- test/upstream-e2e-tests.sh | 5 +++++ 5 files changed, 78 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 729b2c6bad..0e6c7482f7 100644 --- a/Makefile +++ b/Makefile @@ -70,10 +70,18 @@ test-upstream-e2e: UNINSTALL_STRIMZI="false" ./hack/strimzi.sh INSTALL_KAFKA=true TEST_KNATIVE_KAFKA=true ./test/upstream-e2e-tests.sh +# Run upstream E2E tests with net-istio and sidecar. +# TODO: Enable upgrade tests once upstream fixed the issue https://github.com/knative/serving/issues/11535. +test-upstream-e2e-mesh: + UNINSTALL_STRIMZI="false" ./hack/strimzi.sh + FULL_MESH=true INSTALL_KAFKA=false TEST_KNATIVE_KAFKA=false TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh + # Run upstream E2E tests without upgrades. test-upstream-e2e-no-upgrade: UNINSTALL_STRIMZI="false" ./hack/strimzi.sh - INSTALL_KAFKA=true TEST_KNATIVE_KAFKA=true TEST_KNATIVE_E2E=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh + # This is to run with FULL_MESH to verify the PR. Remove this and use "test-upstream-e2e-mesh". + FULL_MESH=true TEST_KNATIVE_KAFKA=false TEST_KNATIVE_E2E=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh + # INSTALL_KAFKA=true TEST_KNATIVE_KAFKA=true TEST_KNATIVE_E2E=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh # Run only upstream upgrade tests. test-upstream-upgrade: @@ -131,4 +139,4 @@ lint: golangci-lint run find . -type f -path './**/*.*sh' -not -path '*vendor*' | xargs -r shellcheck operator-sdk bundle validate ./olm-catalog/serverless-operator - git ls-files | grep -Ev '^(vendor/|.git)' | xargs misspell -error \ No newline at end of file + git ls-files | grep -Ev '^(vendor/|.git)' | xargs misspell -error diff --git a/hack/lib/serverless.bash b/hack/lib/serverless.bash index 19f398a42c..f938cbe284 100644 --- a/hack/lib/serverless.bash +++ b/hack/lib/serverless.bash @@ -153,12 +153,57 @@ function deploy_knativeserving_cr { # This is a way to test backwards compatibility of the product with the older full-blown configuration. oc apply -n "${SERVING_NAMESPACE}" -f "${rootdir}/test/v1alpha1/resources/operator.knative.dev_v1alpha1_knativeserving_cr.yaml" + if [[ $FULL_MESH == "true" ]]; then + enable_net_istio + fi + timeout 900 "[[ \$(oc get knativeserving.operator.knative.dev knative-serving \ -n ${SERVING_NAMESPACE} -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]" logger.success 'Knative Serving has been installed successfully.' } +# enable_net_istio adds patch to KnativeServing: +# - Set ingress.istio.enbled to "true" +# - Set inject and rewriteAppHTTPProbers annotations for activator and autoscaler +# - Override observability.metrics.backend-destination to "none", +# as "test/v1alpha1/resources/operator.knative.dev_v1alpha1_knativeserving_cr.yaml" has the value "prometheus". +function enable_net_istio { + patchfile="$(mktemp -t knative-serving-XXXXX.yaml)" + cat - << EOF > "${patchfile}" +spec: + ingress: + istio: + enabled: true + deployments: + - annotations: + sidecar.istio.io/inject: "true" + sidecar.istio.io/rewriteAppHTTPProbers: "true" + name: activator + - annotations: + sidecar.istio.io/inject: "true" + sidecar.istio.io/rewriteAppHTTPProbers: "true" + name: autoscaler + - name: domain-mapping + replicas: 2 + config: + observability: + metrics.backend-destination: "none" +EOF + + oc patch knativeserving knative-serving \ + -n "${SERVING_NAMESPACE}" \ + --type merge --patch-file="${patchfile}" + + timeout 900 "[[ \$(oc get knativeserving.operator.knative.dev knative-serving \ + -n ${SERVING_NAMESPACE} -o=jsonpath='{.status.conditions[?(@.type==\"Ready\")].status}') != True ]]" + + logger.success 'KnativeServing has been updated successfully.' + + # metadata-webhook adds istio annotations for e2e test by webhook. + oc apply -f https://raw.githubusercontent.com/nak3/metadata-webhook/main/examples/release.yaml +} + function deploy_knativeeventing_cr { logger.info 'Deploy Knative Eventing' diff --git a/hack/lib/vars.bash b/hack/lib/vars.bash index 226d1e2deb..c1ee647788 100644 --- a/hack/lib/vars.bash +++ b/hack/lib/vars.bash @@ -69,3 +69,4 @@ export TEST_KNATIVE_KAFKA="${TEST_KNATIVE_KAFKA:-false}" export INSTALL_SERVING="${INSTALL_SERVING:-true}" export INSTALL_EVENTING="${INSTALL_EVENTING:-true}" export INSTALL_KAFKA="${INSTALL_KAFKA:-false}" +export FULL_MESH="${FULL_MESH:-false}" diff --git a/test/serving.bash b/test/serving.bash index 984279e38e..dd2fb6cb89 100644 --- a/test/serving.bash +++ b/test/serving.bash @@ -48,11 +48,27 @@ function upstream_knative_serving_e2e_and_conformance_tests { image_template="registry.ci.openshift.org/openshift/knative-${KNATIVE_SERVING_VERSION}:knative-serving-test-{{.Name}}" OPENSHIFT_TEST_OPTIONS="--kubeconfig $KUBECONFIG --enable-beta --enable-alpha --resolvabledomain" + if [[ $FULL_MESH == "true" ]]; then + subdomain=$(oc get ingresses.config.openshift.io cluster -o jsonpath="{.spec.domain}") + OPENSHIFT_TEST_OPTIONS+=" --https --customdomain=$subdomain" + + # Use x509ignoreCN=0. + # This should not be necesssary if we could ceate certs with SAN. However, openssl command in the CI + # seems old version and so it does not have "-addext" option to add SAN. + export GODEBUG="x509ignoreCN=0" + + # TODO: SRVKS-211: Can not run grpc and http2 tests. + rm ./test/e2e/grpc_test.go + rm ./test/e2e/http2_test.go + # Remove h2c test + sed -ie '46,50d' ./test/conformance/runtime/protocol_test.go + fi + local parallel=3 if [[ $(oc get infrastructure cluster -ojsonpath='{.status.platform}') = VSphere ]]; then # Since we don't have LoadBalancers working, gRPC tests will always fail. - rm ./test/e2e/grpc_test.go + rm -f ./test/e2e/grpc_test.go parallel=2 fi diff --git a/test/upstream-e2e-tests.sh b/test/upstream-e2e-tests.sh index 6a94ace19b..97fe7b22db 100755 --- a/test/upstream-e2e-tests.sh +++ b/test/upstream-e2e-tests.sh @@ -19,6 +19,11 @@ logger.success '🚀 Cluster prepared for testing.' scale_up_workers +# Install ServiceMesh and enable mTLS. +if [[ $FULL_MESH == true ]]; then + UNINSTALL_MESH="false" install_mesh +fi + # Run upgrade tests if [[ $TEST_KNATIVE_UPGRADE == true ]]; then install_serverless_previous From 03de2e6ee6b19c49718f91fecb3e0466d18a50c3 Mon Sep 17 00:00:00 2001 From: Kenjiro Nakayama Date: Mon, 12 Jul 2021 19:55:48 +0900 Subject: [PATCH 2/2] Revert makefile's temporary change --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0e6c7482f7..2d8baf9ae2 100644 --- a/Makefile +++ b/Makefile @@ -79,9 +79,7 @@ test-upstream-e2e-mesh: # Run upstream E2E tests without upgrades. test-upstream-e2e-no-upgrade: UNINSTALL_STRIMZI="false" ./hack/strimzi.sh - # This is to run with FULL_MESH to verify the PR. Remove this and use "test-upstream-e2e-mesh". - FULL_MESH=true TEST_KNATIVE_KAFKA=false TEST_KNATIVE_E2E=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh - # INSTALL_KAFKA=true TEST_KNATIVE_KAFKA=true TEST_KNATIVE_E2E=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh + INSTALL_KAFKA=true TEST_KNATIVE_KAFKA=true TEST_KNATIVE_E2E=true TEST_KNATIVE_UPGRADE=false ./test/upstream-e2e-tests.sh # Run only upstream upgrade tests. test-upstream-upgrade: