diff --git a/test/README.md b/test/README.md index d84ce912..09d4a76b 100644 --- a/test/README.md +++ b/test/README.md @@ -12,10 +12,10 @@ Before running the integration, please make sure you have installed install custom resource for operator or knative-serving installed in your cluster. -Create a namespace called `operator-tests` if it is missing. +Create a namespace called `knative-serving` if it is missing. ```bash -kubectl create namespace operator-tests +kubectl create namespace knative-serving ``` To run all integration tests: diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 8aa7264c..a4686625 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -17,16 +17,20 @@ # This script provides helper methods to perform cluster actions. source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/e2e-tests.sh -# Latest serving release. This is intentionally hardcoded for now, but -# will need the ability to test against the latest successful serving -# CI runs in the future. -readonly LATEST_SERVING_RELEASE_VERSION=$(git describe --match "v[0-9]*" --abbrev=0) +# Latest serving operator release. +readonly LATEST_SERVING_OPERATOR_RELEASE_VERSION=$(git tag | sort -V | tail -1) +# Latest serving release. This can be different from LATEST_SERVING_OPERATOR_RELEASE_VERSION. +LATEST_SERVING_RELEASE_VERSION="v0.12.1" # Istio version we test with readonly ISTIO_VERSION="1.4.2" # Test without Istio mesh enabled readonly ISTIO_MESH=0 # Namespace used for tests -readonly TEST_NAMESPACE="operator-tests" +readonly TEST_NAMESPACE="knative-serving" + +OPERATOR_DIR=$(dirname $0)/.. +KNATIVE_SERVING_DIR=${OPERATOR_DIR}/.. +release_yaml="$(mktemp)" # Choose a correct istio-crds.yaml file. # - $1 specifies Istio version. @@ -50,6 +54,24 @@ function istio_yaml() { echo "third_party/istio-${istio_version}/istio-${suffix}.yaml" } +# Download the repository of Knative Serving. The purpose of this function is to download the source code of serving +# and retrive the LATEST_SERVING_RELEASE_VERSION for further use. +# Parameter: $1 - branch of the repository. +function donwload_knative_serving() { + # Go the directory to download the source code of knative serving + cd ${KNATIVE_SERVING_DIR} + # Download the source code of knative serving + git clone https://github.com/knative/serving.git + cd serving + local branch=$1 + if [ -n "${branch}" ] ; then + git fetch origin ${branch}:${branch} + git checkout ${branch} + fi + LATEST_SERVING_RELEASE_VERSION=$(git tag | sort -V | tail -1) + cd ${OPERATOR_DIR} +} + # Install Istio. function install_istio() { local base_url="https://raw.githubusercontent.com/knative/serving/${LATEST_SERVING_RELEASE_VERSION}" @@ -59,7 +81,7 @@ function install_istio() { echo ">> Installing Istio" echo "Istio CRD YAML: ${INSTALL_ISTIO_CRD_YAML}" echo "Istio YAML: ${INSTALL_ISTIO_YAML}" - + echo ">> Bringing up Istio" echo ">> Running Istio CRD installer" kubectl apply -f "${INSTALL_ISTIO_CRD_YAML}" || return 1 @@ -76,6 +98,7 @@ function create_namespace() { } function install_serving_operator() { + cd ${OPERATOR_DIR} header "Installing Knative Serving operator" # Deploy the operator ko apply -f config/ @@ -87,7 +110,7 @@ function knative_teardown() { echo ">> Uninstalling Knative serving" echo "Istio YAML: ${INSTALL_ISTIO_YAML}" echo ">> Bringing down Serving" - kubectl delete -n knative-serving knativeserving --all + kubectl delete -n $TEST_NAMESPACE KnativeServing --all echo ">> Bringing down Istio" kubectl delete --ignore-not-found=true -f "${INSTALL_ISTIO_YAML}" || return 1 kubectl delete --ignore-not-found=true clusterrolebinding cluster-admin-binding diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index 0f1c3163..06cbc869 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -29,6 +29,7 @@ source $(dirname $0)/e2e-common.sh function knative_setup() { + donwload_knative_serving install_istio || fail_test "Istio installation failed" create_namespace install_serving_operator diff --git a/test/e2e-upgrade-tests.sh b/test/e2e-upgrade-tests.sh index 430d6e58..22734116 100755 --- a/test/e2e-upgrade-tests.sh +++ b/test/e2e-upgrade-tests.sh @@ -33,21 +33,21 @@ source $(dirname $0)/e2e-common.sh -OPERATOR_DIR=$(dirname $0)/.. -KNATIVE_SERVING_DIR=${OPERATOR_DIR}/.. +function install_previous_operator_release() { + local full_url="https://github.com/knative/serving-operator/releases/download/${LATEST_SERVING_OPERATOR_RELEASE_VERSION}/serving-operator.yaml" -function install_latest_operator_release() { - header "Installing Knative Serving operator latest public release" - local full_url="https://github.com/knative/serving-operator/releases/download/${LATEST_SERVING_RELEASE_VERSION}/serving-operator.yaml" - - local release_yaml="$(mktemp)" wget "${full_url}" -O "${release_yaml}" \ || fail_test "Unable to download latest Knative Serving Operator release." + donwload_knative_serving install_istio || fail_test "Istio installation failed" + install_previous_serving_release +} + +function install_previous_serving_release() { + header "Installing Knative Serving operator previous public release" kubectl apply -f "${release_yaml}" || fail_test "Knative Serving Operator latest release installation failed" - create_custom_resource - wait_until_pods_running ${TEST_NAMESPACE} + wait_until_pods_running default || fail_test "Serving Operator did not come up" } function create_custom_resource() { @@ -73,31 +73,51 @@ EOF function knative_setup() { create_namespace - install_latest_operator_release + install_previous_operator_release + create_custom_resource + wait_until_pods_running ${TEST_NAMESPACE} } -function install_head() { +# Create test resources and images +function test_setup() { generate_latest_serving_manifest - install_serving_operator + echo ">> Creating test resources (test/config/) in Knative Serving repository" + cd ${KNATIVE_SERVING_DIR}/serving + ko apply ${KO_FLAGS} -f test/config/ || return 1 + + echo ">> Uploading test images..." + # We only need to build and publish two images among all the test images + ${OPERATOR_DIR}/test/upload-test-images.sh ${KNATIVE_SERVING_DIR}/serving "test/test_images/pizzaplanetv1" + ${OPERATOR_DIR}/test/upload-test-images.sh ${KNATIVE_SERVING_DIR}/serving "test/test_images/pizzaplanetv2" + + echo ">> Waiting for Ingress provider to be running..." + if [[ -n "${ISTIO_VERSION}" ]]; then + wait_until_pods_running istio-system || return 1 + wait_until_service_has_external_ip istio-system istio-ingressgateway + fi + cd ${OPERATOR_DIR} } +# This function either generate the manifest based on a branch or download the latest manifest for Knative Serving. +# Parameter: $1 - branch name. If it is empty, download the manifest from nightly build. function generate_latest_serving_manifest() { - # Go the directory to download the source code of knative serving - cd ${KNATIVE_SERVING_DIR} - - # Download the source code of knative serving - git clone https://github.com/knative/serving.git - cd serving - COMMIT_ID=$(git rev-parse --verify HEAD) - echo ">> The latest commit ID of Knative Serving is ${COMMIT_ID}." + cd ${KNATIVE_SERVING_DIR}/serving mkdir -p output + local branch=$1 + if [[ -n "${branch}" ]]; then + git checkout ${branch} + COMMIT_ID=$(git rev-parse --verify HEAD) + echo ">> The latest commit ID of Knative Serving is ${COMMIT_ID}." + # Generate the manifest + export YAML_OUTPUT_DIR=${KNATIVE_SERVING_DIR}/serving/output + ./hack/generate-yamls.sh ${KNATIVE_SERVING_DIR}/serving ${YAML_OUTPUT_DIR}/output.yaml + else + echo ">> Download the latest nightly build of Knative Serving." + # Download the latest manifest + SERVING_YAML=${KNATIVE_SERVING_DIR}/serving/output/serving.yaml + wget -O ${SERVING_YAML} https://storage.googleapis.com/knative-nightly/serving/latest/serving.yaml + fi - # Generate the manifest - export YAML_OUTPUT_DIR=${KNATIVE_SERVING_DIR}/serving/output - ./hack/generate-yamls.sh ${KNATIVE_SERVING_DIR}/serving ${YAML_OUTPUT_DIR}/output.yaml - - # Copy the serving.yaml into cmd/manager/kodata/knative-serving - SERVING_YAML=${KNATIVE_SERVING_DIR}/serving/output/serving.yaml if [[ -f "${SERVING_YAML}" ]]; then echo ">> Replacing the current manifest in operator with the generated manifest" rm -rf ${OPERATOR_DIR}/cmd/manager/kodata/knative-serving/* @@ -115,14 +135,49 @@ initialize $@ --skip-istio-addon TIMEOUT=20m -install_head +header "Running preupgrade tests" + +cd ${KNATIVE_SERVING_DIR}/serving +go_test_e2e -tags=preupgrade -timeout=${TIMEOUT} ./test/upgrade \ + --resolvabledomain="false" "--https" || fail_test + +# Remove this in case we failed to clean it up in an earlier test. +rm -f /tmp/prober-signal + +go_test_e2e -tags=probe -timeout=${TIMEOUT} ./test/upgrade \ + --resolvabledomain="false" "--https" & +PROBER_PID=$! +echo "Prober PID is ${PROBER_PID}" + +install_serving_operator # If we got this far, the operator installed Knative Serving of the latest source code. header "Running tests for Knative Serving Operator" failed=0 -# Run the postupgrade tests +# Run the postupgrade tests under operator +# Operator tests here will make sure that all the Knative deployments reach the desired states and operator CR is +# in ready state. +cd ${OPERATOR_DIR} go_test_e2e -tags=postupgrade -timeout=${TIMEOUT} ./test/upgrade || failed=1 +wait_until_pods_running ${TEST_NAMESPACE} + +header "Running tests under Knative Serving" +# Run the postupgrade tests under serving +cd ${KNATIVE_SERVING_DIR}/serving +go_test_e2e -tags=postupgrade -timeout=${TIMEOUT} ./test/upgrade || failed=1 + +install_previous_serving_release +wait_until_pods_running ${TEST_NAMESPACE} + +header "Running postdowngrade tests" +go_test_e2e -tags=postdowngrade -timeout=${TIMEOUT} ./test/upgrade \ + --resolvabledomain="false" || fail_test + +echo "done" > /tmp/prober-signal + +header "Waiting for prober test" +wait ${PROBER_PID} || fail_test "Prober failed" # Require that tests succeeded. (( failed )) && fail_test diff --git a/test/e2e_flags.go b/test/e2e_flags.go index e6b8b102..a790e9d3 100644 --- a/test/e2e_flags.go +++ b/test/e2e_flags.go @@ -23,7 +23,7 @@ import "os" var ( // ServingOperatorNamespace is the default namespace for serving operator e2e tests - ServingOperatorNamespace = getenv("TEST_NAMESPACE", "operator-tests") + ServingOperatorNamespace = getenv("TEST_NAMESPACE", "knative-serving") // ServingOperatorName is the default operator name for serving operator e2e tests ServingOperatorName = getenv("TEST_RESOURCE", "knative-serving") ) diff --git a/test/resources/knativeserving.go b/test/resources/knativeserving.go index c438b632..1102d181 100644 --- a/test/resources/knativeserving.go +++ b/test/resources/knativeserving.go @@ -129,3 +129,56 @@ func getTestKSOperatorCRSpec() v1alpha1.KnativeServingSpec { }, } } + +// WaitForKnativeServingDeploymentState polls the status of the Knative deployments every `interval` +// until `inState` returns `true` indicating the deployments match the desired deployments. +func WaitForKnativeServingDeploymentState(clients *test.Clients, namespace string, expectedDeployments []string, + inState func(deps *v1.DeploymentList, expectedDeployments []string, err error) (bool, error)) (*v1alpha1.KnativeServing, error) { + span := logging.GetEmitableSpan(context.Background(), fmt.Sprintf("WaitForKnativeDeploymentState/%s/%s", expectedDeployments, "KnativeDeploymentIsReady")) + defer span.End() + + var lastState *v1alpha1.KnativeServing + waitErr := wait.PollImmediate(Interval, Timeout, func() (bool, error) { + dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(namespace).List(metav1.ListOptions{}) + return inState(dpList, expectedDeployments, err) + }) + + if waitErr != nil { + return lastState, waitErr + } + return lastState, nil +} + + +// IsKnativeServingDeploymentReady will check the status conditions of the deployments and return true if the deployments meet the desired status. +func IsKnativeServingDeploymentReady(dpList *v1.DeploymentList, expectedDeployments []string, err error) (bool, error) { + if err != nil { + return false, err + } + if len(dpList.Items) != len(expectedDeployments) { + errMessage := fmt.Sprintf("The expected number of deployments is %v, and got %v.", len(expectedDeployments), len(dpList.Items)) + return false, errors.New(errMessage) + } + for _, deployment := range dpList.Items { + if !stringInList(deployment.Name, expectedDeployments) { + errMessage := fmt.Sprintf("The deployment %v is not found in the expected list of deployment.", deployment.Name) + return false, errors.New(errMessage) + } + for _, c := range deployment.Status.Conditions { + if c.Type == v1.DeploymentAvailable && c.Status != corev1.ConditionTrue { + errMessage := fmt.Sprintf("The deployment %v is not ready.", deployment.Name) + return false, errors.New(errMessage) + } + } + } + return true, nil +} + +func stringInList(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} diff --git a/test/resources/verify.go b/test/resources/verify.go index 8c39ea5c..eb0afc49 100644 --- a/test/resources/verify.go +++ b/test/resources/verify.go @@ -237,3 +237,11 @@ func verifyNoKSOperatorCR(clients *test.Clients) error { } return nil } + +// AssertKSOperatorDeploymentStatus verifies if the Knative deployments reach the READY status. +func AssertKSOperatorDeploymentStatus(t *testing.T, clients *test.Clients, namespace string, expectedDeployments []string) { + if _, err := WaitForKnativeServingDeploymentState(clients, namespace, expectedDeployments, + IsKnativeServingDeploymentReady); err != nil { + t.Fatalf("Knative Serving deployments failed to meet the expected deployments: %v", err) + } +} diff --git a/test/upgrade/servingoperator_postupgrade_test.go b/test/upgrade/servingoperator_postupgrade_test.go index b4221c40..47d1d494 100644 --- a/test/upgrade/servingoperator_postupgrade_test.go +++ b/test/upgrade/servingoperator_postupgrade_test.go @@ -18,8 +18,6 @@ package e2e import ( "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/test/logstream" "knative.dev/serving-operator/test" "knative.dev/serving-operator/test/client" @@ -38,60 +36,17 @@ func TestKnativeServingPostUpgrade(t *testing.T) { Namespace: test.ServingOperatorNamespace, } - test.CleanupOnInterrupt(func() { test.TearDown(clients, names) }) - defer test.TearDown(clients, names) - // Create a KnativeServing custom resource, if it does not exist if _, err := resources.EnsureKnativeServingExists(clients.KnativeServing(), names); err != nil { t.Fatalf("KnativeService %q failed to create: %v", names.KnativeServing, err) } - // Test if KnativeServing can reach the READY status after upgrade - t.Run("create", func(t *testing.T) { - resources.AssertKSOperatorCRReadyStatus(t, clients, names) - }) - // Verify if resources match the latest requirement after upgrade t.Run("verify resources", func(t *testing.T) { - resources.AssertKSOperatorCRReadyStatus(t, clients, names) // TODO: We only verify the deployment, but we need to add other resources as well, like ServiceAccount, ClusterRoleBinding, etc. expectedDeployments := []string{"networking-istio", "webhook", "controller", "activator", "autoscaler-hpa", "autoscaler"} - ksVerifyDeployment(t, clients, names, expectedDeployments) - }) - - // TODO: We will add one or sections here to run the tests tagged with postupgrade in knative serving. - - // Delete the KnativeServing to see if all resources will be removed after upgrade - t.Run("delete", func(t *testing.T) { + resources.AssertKSOperatorDeploymentStatus(t, clients, names.Namespace, expectedDeployments) resources.AssertKSOperatorCRReadyStatus(t, clients, names) - resources.KSOperatorCRDelete(t, clients, names) }) } - -// ksVerifyDeployment verify whether the deployments have the correct number and names. -func ksVerifyDeployment(t *testing.T, clients *test.Clients, names test.ResourceNames, - expectedDeployments []string) { - dpList, err := clients.KubeClient.Kube.AppsV1().Deployments(names.Namespace).List(metav1.ListOptions{}) - assertEqual(t, err, nil) - assertEqual(t, len(dpList.Items), len(expectedDeployments)) - for _, deployment := range dpList.Items { - assertEqual(t, stringInList(deployment.Name, expectedDeployments), true) - } -} - -func assertEqual(t *testing.T, actual, expected interface{}) { - if actual == expected { - return - } - t.Fatalf("expected does not equal actual. \nExpected: %v\nActual: %v", expected, actual) -} - -func stringInList(a string, list []string) bool { - for _, b := range list { - if b == a { - return true - } - } - return false -} diff --git a/test/upload-test-images.sh b/test/upload-test-images.sh new file mode 100755 index 00000000..041162db --- /dev/null +++ b/test/upload-test-images.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Copyright 2018 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit + +# This fucntion is used to build and publish the test images. +# $1 - the first parameter is a string, specifying a path of the root directory of the project to run the ko command. +# $2 - the second parameter is a string, speficying a sub-directory, where the images are built, under the root directory. +# $3 - the third parameter is a string, speficying the tag of the images. +function upload_test_images() { + echo ">> Publishing test images" + # Script needs to be executed from the root directory + # to pickup .ko.yaml + local root_dir=$1 + if [ ! -n "$root_dir" ] ; then + root_dir="$( dirname "$0")/.." + fi + cd ${root_dir} + local image_dir=$2 + if [ ! -n "$image_dir" ] ; then + image_dir="test/test_images" + fi + local docker_tag=$3 + local tag_option="" + if [ -n "${docker_tag}" ]; then + tag_option="--tags $docker_tag,latest" + fi + + # ko resolve is being used for the side-effect of publishing images, + # so the resulting yaml produced is ignored. + ko resolve ${tag_option} -RBf "${image_dir}" > /dev/null +} + +: ${KO_DOCKER_REPO:?"You must set 'KO_DOCKER_REPO', see DEVELOPMENT.md"} + +upload_test_images $@