diff --git a/hack/generate-yamls.sh b/hack/generate-yamls.sh index 141adf46264c..3d306c09c84d 100755 --- a/hack/generate-yamls.sh +++ b/hack/generate-yamls.sh @@ -66,6 +66,7 @@ readonly CONSOLIDATED_ARTIFACTS # Flags for all ko commands KO_YAML_FLAGS="-P" +KO_FLAGS="${KO_FLAGS:-}" [[ "${KO_DOCKER_REPO}" != gcr.io/* ]] && KO_YAML_FLAGS="" if [[ "${KO_FLAGS}" != *"--platform"* ]]; then @@ -74,7 +75,7 @@ fi readonly KO_YAML_FLAGS="${KO_YAML_FLAGS} ${KO_FLAGS}" -if [[ -n "${TAG}" ]]; then +if [[ -n "${TAG:-}" ]]; then LABEL_YAML_CMD=(sed -e "s|serving.knative.dev/release: devel|serving.knative.dev/release: \"${TAG}\"|") else LABEL_YAML_CMD=(cat) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 1bc3da3669ec..1e00afed06f6 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -15,8 +15,9 @@ # limitations under the License. # This script provides helper methods to perform cluster actions. -source $(dirname $0)/../vendor/knative.dev/hack/e2e-tests.sh -source $(dirname $0)/e2e-networking-library.sh +# shellcheck disable=SC1090 +source "$(dirname "${BASH_SOURCE[0]}")/../vendor/knative.dev/hack/e2e-tests.sh" +source "$(dirname "${BASH_SOURCE[0]}")/e2e-networking-library.sh" CERT_MANAGER_VERSION="latest" # Since default is istio, make default ingress as istio @@ -39,12 +40,12 @@ MESH=0 INSTALL_CUSTOM_YAMLS="" UNINSTALL_LIST=() -TMP_DIR=$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX) -readonly TMP_DIR +export TMP_DIR +TMP_DIR="${TMP_DIR:-$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX)}" readonly KNATIVE_DEFAULT_NAMESPACE="knative-serving" # This the namespace used to install Knative Serving. Use generated UUID as namespace. export SYSTEM_NAMESPACE -SYSTEM_NAMESPACE=$(uuidgen | tr 'A-Z' 'a-z') +SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}" # Keep this in sync with test/ha/ha.go @@ -52,6 +53,14 @@ readonly REPLICAS=3 readonly BUCKETS=10 HA_COMPONENTS=() +# Latest serving release. If user does not supply this as a flag, the latest +# tagged release on the current branch will be used. +LATEST_SERVING_RELEASE_VERSION=$(latest_version) + +# Latest net-istio release. +LATEST_NET_ISTIO_RELEASE_VERSION=$( + curl -L --silent "https://api.github.com/repos/knative/net-istio/releases" | grep '"tag_name"' \ + | cut -f2 -d: | sed "s/[^v0-9.]//g" | sort | tail -n1) # Parse our custom flags. function parse_flags() { @@ -141,14 +150,15 @@ function parse_flags() { # All generated YAMLs will be available and pointed by the corresponding # environment variables as set in /hack/generate-yamls.sh. function build_knative_from_source() { - local YAML_LIST="$(mktemp)" + local FULL_OUTPUT YAML_LIST LOG_OUTPUT ENV_OUTPUT + YAML_LIST="$(mktemp)" # Generate manifests, capture environment variables pointing to the YAML files. - local FULL_OUTPUT="$( \ - source $(dirname $0)/../hack/generate-yamls.sh ${REPO_ROOT_DIR} ${YAML_LIST} ; \ + FULL_OUTPUT="$( \ + source "$(dirname "${BASH_SOURCE[0]}")/../hack/generate-yamls.sh" "${REPO_ROOT_DIR}" "${YAML_LIST}" ; \ set | grep _YAML=/)" - local LOG_OUTPUT="$(echo "${FULL_OUTPUT}" | grep -v _YAML=/)" - local ENV_OUTPUT="$(echo "${FULL_OUTPUT}" | grep '^[_0-9A-Z]\+_YAML=/')" + LOG_OUTPUT="$(echo "${FULL_OUTPUT}" | grep -v _YAML=/)" + ENV_OUTPUT="$(echo "${FULL_OUTPUT}" | grep '^[_0-9A-Z]\+_YAML=/')" [[ -z "${LOG_OUTPUT}" || -z "${ENV_OUTPUT}" ]] && fail_test "Error generating manifests" # Only import the environment variables pointing to the YAML files. echo "${LOG_OUTPUT}" @@ -164,7 +174,7 @@ function build_knative_from_source() { function install_knative_serving() { local version=${1:-"HEAD"} if [[ -z "${INSTALL_CUSTOM_YAMLS}" ]]; then - install_knative_serving_standard "$version" "$2" + install_knative_serving_standard "$version" "${2:-}" return fi echo ">> Installing Knative serving from custom YAMLs" @@ -227,29 +237,31 @@ function install_knative_serving_standard() { ko apply -f "${SERVING_RELEASE_YAML}" --selector=knative.dev/crd-install=true || return 1 fi - echo ">> Installing Ingress" - if [[ -n "${GLOO_VERSION}" ]]; then - install_gloo || return 1 - elif [[ -n "${KOURIER_VERSION}" ]]; then - install_kourier || return 1 - elif [[ -n "${AMBASSADOR_VERSION}" ]]; then - install_ambassador || return 1 - elif [[ -n "${CONTOUR_VERSION}" ]]; then - install_contour || return 1 - elif [[ -n "${KONG_VERSION}" ]]; then - install_kong || return 1 - else - if [[ "$1" == "HEAD" ]]; then - install_istio "./third_party/istio-latest/net-istio.yaml" || return 1 + if [[ -z "${REUSE_INGRESS:-}" ]]; then + echo ">> Installing Ingress" + if [[ -n "${GLOO_VERSION:-}" ]]; then + install_gloo || return 1 + elif [[ -n "${KOURIER_VERSION:-}" ]]; then + install_kourier || return 1 + elif [[ -n "${AMBASSADOR_VERSION:-}" ]]; then + install_ambassador || return 1 + elif [[ -n "${CONTOUR_VERSION:-}" ]]; then + install_contour || return 1 + elif [[ -n "${KONG_VERSION:-}" ]]; then + install_kong || return 1 else - # Download the latest release of net-istio. - local url="https://github.com/knative/net-istio/releases/download/${LATEST_NET_ISTIO_RELEASE_VERSION}" - local yaml="net-istio.yaml" - local YAML_NAME=${TMP_DIR}/"net-istio-${LATEST_NET_ISTIO_RELEASE_VERSION}.yaml" - wget "${url}/${yaml}" -O "${YAML_NAME}" \ - || fail_test "Unable to download latest knative/net-istio release." - echo "net-istio YAML: ${YAML_NAME}" - install_istio $YAML_NAME || return 1 + if [[ "$1" == "HEAD" ]]; then + install_istio "./third_party/istio-latest/net-istio.yaml" || return 1 + else + # Download the latest release of net-istio. + local url="https://github.com/knative/net-istio/releases/download/${LATEST_NET_ISTIO_RELEASE_VERSION}" + local yaml="net-istio.yaml" + local YAML_NAME=${TMP_DIR}/"net-istio-${LATEST_NET_ISTIO_RELEASE_VERSION}.yaml" + wget "${url}/${yaml}" -O "${YAML_NAME}" \ + || fail_test "Unable to download latest knative/net-istio release." + echo "net-istio YAML: ${YAML_NAME}" + install_istio $YAML_NAME || return 1 + fi fi fi @@ -511,3 +523,27 @@ function disable_chaosduck() { function enable_chaosduck() { kubectl -n "${SYSTEM_NAMESPACE}" scale deployment "chaosduck" --replicas=1 || fail_test } + +function install_latest_release() { + header "Installing Knative latest public release" + + install_knative_serving latest-release \ + || fail_test "Knative latest release installation failed" + test_logging_config_setup + wait_until_pods_running ${SYSTEM_NAMESPACE} + wait_until_batch_job_complete ${SYSTEM_NAMESPACE} +} + +function install_head_reuse_ingress() { + header "Installing Knative head release and reusing ingress" + # Keep the existing ingress and do not upgrade it. The ingress upgrade + # makes ongoing requests fail. + REUSE_INGRESS=true install_knative_serving || fail_test "Knative head release installation failed" + test_logging_config_setup + wait_until_pods_running ${SYSTEM_NAMESPACE} + wait_until_batch_job_complete ${SYSTEM_NAMESPACE} +} + +function knative_setup() { + install_latest_release +} diff --git a/test/e2e-networking-library.sh b/test/e2e-networking-library.sh index 635cab9fab69..972f0148fa8e 100644 --- a/test/e2e-networking-library.sh +++ b/test/e2e-networking-library.sh @@ -15,17 +15,17 @@ # limitations under the License. function install_istio() { - if [[ -z "${ISTIO_VERSION}" ]]; then + if [[ -z "${ISTIO_VERSION:-}" ]]; then readonly ISTIO_VERSION="stable" fi - if [[ -z "${NET_ISTIO_COMMIT}" ]]; then + if [[ -z "${NET_ISTIO_COMMIT:-}" ]]; then NET_ISTIO_COMMIT=$(head -n 1 ${1} | grep "# Generated when HEAD was" | sed 's/^.* //') echo "Got NET_ISTIO_COMMIT from ${1}: ${NET_ISTIO_COMMIT}" fi # TODO: remove this when all the net-istio.yaml in use contain a commit ID - if [[ -z "${NET_ISTIO_COMMIT}" ]]; then + if [[ -z "${NET_ISTIO_COMMIT:-}" ]]; then NET_ISTIO_COMMIT="8102cd3d32f05be1c58260a9717d532a4a6d2f60" echo "Hard coded NET_ISTIO_COMMIT: ${NET_ISTIO_COMMIT}" fi @@ -41,7 +41,7 @@ function install_istio() { ) ISTIO_PROFILE="istio" - if [[ -n "$KIND" ]]; then + if [[ -n "${KIND:-}" ]]; then ISTIO_PROFILE+="-kind" else ISTIO_PROFILE+="-ci" @@ -52,7 +52,7 @@ function install_istio() { ISTIO_PROFILE+="-mesh" ISTIO_PROFILE+=".yaml" - if [[ -n "$CLUSTER_DOMAIN" ]]; then + if [[ -n "${CLUSTER_DOMAIN:-}" ]]; then sed -ie "s/cluster\.local/${CLUSTER_DOMAIN}/g" ${NET_ISTIO_DIR}/third_party/istio-${ISTIO_VERSION}/${ISTIO_PROFILE} fi @@ -61,7 +61,7 @@ function install_istio() { echo "Istio profile: ${ISTIO_PROFILE}" ${NET_ISTIO_DIR}/third_party/istio-${ISTIO_VERSION}/install-istio.sh ${ISTIO_PROFILE} - if [[ -n "$1" ]]; then + if [[ -n "${1:-}" ]]; then echo ">> Installing net-istio" echo "net-istio original YAML: ${1}" # Create temp copy in which we replace knative-serving by the test's system namespace. @@ -161,11 +161,11 @@ function install_contour() { } function wait_until_ingress_running() { - if [[ -n "${ISTIO_VERSION}" ]]; then + if [[ -n "${ISTIO_VERSION:-}" ]]; then wait_until_pods_running istio-system || return 1 wait_until_service_has_external_http_address istio-system istio-ingressgateway || return 1 fi - if [[ -n "${GLOO_VERSION}" ]]; then + if [[ -n "${GLOO_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Gloo # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=knative-external-proxy @@ -173,7 +173,7 @@ function wait_until_ingress_running() { wait_until_pods_running gloo-system || return 1 wait_until_service_has_external_ip gloo-system knative-external-proxy fi - if [[ -n "${KOURIER_VERSION}" ]]; then + if [[ -n "${KOURIER_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Kourier # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=kourier @@ -181,7 +181,7 @@ function wait_until_ingress_running() { wait_until_pods_running kourier-system || return 1 wait_until_service_has_external_http_address kourier-system kourier fi - if [[ -n "${AMBASSADOR_VERSION}" ]]; then + if [[ -n "${AMBASSADOR_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Ambassador # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=ambassador @@ -189,7 +189,7 @@ function wait_until_ingress_running() { wait_until_pods_running ambassador || return 1 wait_until_service_has_external_http_address ambassador ambassador fi - if [[ -n "${CONTOUR_VERSION}" ]]; then + if [[ -n "${CONTOUR_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Contour # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=envoy @@ -198,7 +198,7 @@ function wait_until_ingress_running() { wait_until_pods_running contour-internal || return 1 wait_until_service_has_external_ip "${GATEWAY_NAMESPACE_OVERRIDE}" "${GATEWAY_OVERRIDE}" fi - if [[ -n "${KONG_VERSION}" ]]; then + if [[ -n "${KONG_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Kong # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=kong-proxy diff --git a/test/e2e-upgrade-tests.sh b/test/e2e-upgrade-tests.sh index 5c37169f5868..35fc8e5ea157 100755 --- a/test/e2e-upgrade-tests.sh +++ b/test/e2e-upgrade-tests.sh @@ -28,39 +28,8 @@ # You can specify the version to run against with the --version argument # (e.g. --version v0.7.0). If this argument is not specified, the script will # run against the latest tagged version on the current branch. - -source $(dirname $0)/e2e-common.sh - -# Latest serving release. If user does not supply this as a flag, the latest -# tagged release on the current branch will be used. -LATEST_SERVING_RELEASE_VERSION=$(latest_version) - -# Latest net-istio release. -LATEST_NET_ISTIO_RELEASE_VERSION=$( - curl -L --silent "https://api.github.com/repos/knative/net-istio/releases" | grep '"tag_name"' \ - | cut -f2 -d: | sed "s/[^v0-9.]//g" | sort | tail -n1) - -function install_latest_release() { - header "Installing Knative latest public release" - - install_knative_serving latest-release \ - || fail_test "Knative latest release installation failed" - test_logging_config_setup - wait_until_pods_running ${SYSTEM_NAMESPACE} - wait_until_batch_job_complete ${SYSTEM_NAMESPACE} -} - -function install_head() { - header "Installing Knative head release" - install_knative_serving || fail_test "Knative head release installation failed" - test_logging_config_setup - wait_until_pods_running ${SYSTEM_NAMESPACE} - wait_until_batch_job_complete ${SYSTEM_NAMESPACE} -} - -function knative_setup() { - install_latest_release -} +# shellcheck disable=SC1090 +source "$(dirname "${BASH_SOURCE[0]}")/e2e-common.sh" # Script entry point. @@ -75,51 +44,14 @@ initialize "$@" --skip-istio-addon --min-nodes=4 --max-nodes=4 disable_chaosduck # TODO(#2656): Reduce the timeout after we get this test to consistently passing. -TIMEOUT=10m -# Probe tests starts before postupgrade tests and ends after postdowngrade tests. -# The timeout should be at least 10m + 10m + installation time -PROBE_TIMEOUT=20m +TIMEOUT=30m -header "Running preupgrade tests" +header "Running upgrade tests" -go_test_e2e -tags=preupgrade -timeout=${TIMEOUT} ./test/upgrade \ +go_test_e2e -tags=upgrade -timeout=${TIMEOUT} \ + ./test/upgrade \ --resolvabledomain=$(use_resolvable_domain) || fail_test -header "Starting prober test" - -# Remove the following files in case we failed to clean them up in an earlier test. -rm -f /tmp/prober-signal -rm -f /tmp/autoscaling-signal -rm -f /tmp/autoscaling-tbc-signal - -go_test_e2e -tags=probe -timeout=${PROBE_TIMEOUT} ./test/upgrade \ - --resolvabledomain=$(use_resolvable_domain) & -PROBER_PID=$! -echo "Prober PID is ${PROBER_PID}" - -install_head - -header "Running postupgrade tests" -go_test_e2e -tags=postupgrade -timeout=${TIMEOUT} ./test/upgrade \ - --resolvabledomain=$(use_resolvable_domain) || fail_test - -install_latest_release - -header "Running postdowngrade tests" -go_test_e2e -tags=postdowngrade -timeout=${TIMEOUT} ./test/upgrade \ - --resolvabledomain=$(use_resolvable_domain) || fail_test - -# The probe tests are blocking on the following files to know when it should exit. -# -# This is kind of gross. First attempt was to just send a signal to the go test, -# but "go test" intercepts the signal and always exits with a non-zero code. -echo "done" > /tmp/prober-signal -echo "done" > /tmp/autoscaling-signal -echo "done" > /tmp/autoscaling-tbc-signal - -header "Waiting for prober test" -wait ${PROBER_PID} || fail_test "Prober failed" - # Remove the kail log file if the test flow passes. # This is for preventing too many large log files to be uploaded to GCS in CI. rm "${KAIL_LOG_FILE}" diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index 7f1b1e5762f2..c3a6d1d1309b 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "knative.dev/pkg/test/logging" + vegeta "github.com/tsenart/vegeta/v12/lib" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" @@ -60,8 +62,9 @@ const ( // TestContext includes context for autoscaler testing. type TestContext struct { t *testing.T + logf logging.FormatLogger clients *test.Clients - names test.ResourceNames + names *test.ResourceNames resources *v1test.ResourceObjects targetUtilization float64 targetValue int @@ -84,15 +87,20 @@ func (ctx *TestContext) SetResources(resources *v1test.ResourceObjects) { } // Names returns the resource names of the TestContext. -func (ctx *TestContext) Names() test.ResourceNames { +func (ctx *TestContext) Names() *test.ResourceNames { return ctx.names } // SetNames set the resource names of the TestContext to the given values. -func (ctx *TestContext) SetNames(names test.ResourceNames) { +func (ctx *TestContext) SetNames(names *test.ResourceNames) { ctx.names = names } +// SetLogger sets the logger of the TestContext. +func (ctx *TestContext) SetLogger(logf logging.FormatLogger) { + ctx.logf = logf +} + func getVegetaTarget(kubeClientset kubernetes.Interface, domain, endpointOverride string, resolvable bool) (vegeta.Target, error) { if resolvable { return vegeta.Target{ @@ -141,7 +149,7 @@ func generateTraffic( for { select { case <-stopChan: - ctx.t.Log("Stopping generateTraffic") + ctx.logf("Stopping generateTraffic") successRate := float64(1) if totalRequests > 0 { successRate = float64(successfulRequests) / float64(totalRequests) @@ -153,14 +161,14 @@ func generateTraffic( return nil case res, ok := <-results: if !ok { - ctx.t.Log("Time is up; done") + ctx.logf("Time is up; done") return nil } totalRequests++ if res.Code != http.StatusOK { - ctx.t.Logf("Status = %d, want: 200", res.Code) - ctx.t.Logf("URL: %s Duration: %v Error: %s Body:\n%s", res.URL, res.Latency, res.Error, string(res.Body)) + ctx.logf("Status = %d, want: 200", res.Code) + ctx.logf("URL: %s Duration: %v Error: %s Body:\n%s", res.URL, res.Latency, res.Error, string(res.Body)) continue } successfulRequests++ @@ -175,7 +183,7 @@ func generateTrafficAtFixedConcurrency(ctx *TestContext, concurrency int, stopCh vegeta.Workers(uint64(concurrency)), vegeta.MaxWorkers(uint64(concurrency))) - ctx.t.Logf("Maintaining %d concurrent requests.", concurrency) + ctx.logf("Maintaining %d concurrent requests.", concurrency) return generateTraffic(ctx, attacker, pacer, stopChan) } @@ -183,7 +191,7 @@ func generateTrafficAtFixedRPS(ctx *TestContext, rps int, stopChan chan struct{} pacer := vegeta.ConstantPacer{Freq: rps, Per: time.Second} attacker := vegeta.NewAttacker(vegeta.Timeout(0)) // No timeout is enforced at all. - ctx.t.Logf("Maintaining %v RPS.", rps) + ctx.logf("Maintaining %v RPS.", rps) return generateTraffic(ctx, attacker, pacer, stopChan) } @@ -194,19 +202,16 @@ func toPercentageString(f float64) string { // SetupSvc creates a new service, with given service options. // It returns a TestContext that has resources, K8s clients and other needed // data points. -// It sets up EnsureTearDown to ensure that resources are cleaned up when the -// test terminates. func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization float64, fopts ...rtesting.ServiceOption) *TestContext { t.Helper() clients := Setup(t) t.Log("Creating a new Route and Configuration") - names := test.ResourceNames{ + names := &test.ResourceNames{ Service: test.ObjectNameForTest(t), Image: autoscaleTestImageName, } - test.EnsureTearDown(t, clients, &names) - resources, err := v1test.CreateServiceReady(t, clients, &names, + resources, err := v1test.CreateServiceReady(t, clients, names, append([]rtesting.ServiceOption{ rtesting.WithConfigAnnotations(map[string]string{ autoscaling.ClassAnnotationKey: class, @@ -249,6 +254,7 @@ func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization return &TestContext{ t: t, + logf: t.Logf, clients: clients, names: names, resources: resources, @@ -265,7 +271,7 @@ func assertScaleDown(ctx *TestContext) { } // Account for the case where scaling up uses all available pods. - ctx.t.Log("Wait for all pods to terminate.") + ctx.logf("Wait for all pods to terminate.") if err := pkgTest.WaitForPodListState( context.Background(), @@ -284,12 +290,12 @@ func assertScaleDown(ctx *TestContext) { ctx.t.Fatalf("Waiting for Pod.List to have no non-Evicted pods of %q: %v", deploymentName, err) } - ctx.t.Log("The Revision should remain ready after scaling to zero.") + ctx.logf("The Revision should remain ready after scaling to zero.") if err := v1test.CheckRevisionState(ctx.clients.ServingClient, ctx.names.Revision, v1test.IsRevisionReady); err != nil { ctx.t.Fatalf("The Revision %s did not stay Ready after scaling down to zero: %v", ctx.names.Revision, err) } - ctx.t.Log("Scaled down.") + ctx.logf("Scaled down.") } func numberOfReadyPods(ctx *TestContext) (float64, error) { @@ -297,11 +303,11 @@ func numberOfReadyPods(ctx *TestContext) (float64, error) { n := ctx.resources.Revision.Name sks, err := ctx.clients.NetworkingClient.ServerlessServices.Get(context.Background(), n, metav1.GetOptions{}) if err != nil { - ctx.t.Logf("Error getting SKS %q: %v", n, err) + ctx.logf("Error getting SKS %q: %v", n, err) return 0, fmt.Errorf("error retrieving sks %q: %w", n, err) } if sks.Status.PrivateServiceName == "" { - ctx.t.Logf("SKS %s has not yet reconciled", n) + ctx.logf("SKS %s has not yet reconciled", n) // Not an error, but no pods either. return 0, nil } @@ -328,7 +334,7 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done return err } mes := fmt.Sprintf("revision %q #replicas: %v, want at least: %v", ctx.resources.Revision.Name, got, minPods) - ctx.t.Log(mes) + ctx.logf(mes) // verify that the number of pods doesn't go down while we are scaling up. if got < minPods { return errors.New("interim scale didn't fulfill constraints: " + mes) @@ -336,7 +342,7 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done // A quick test succeeds when the number of pods scales up to `targetPods` // (and, as an extra check, no more than `maxPods`). if quick && got >= targetPods && got <= maxPods { - ctx.t.Logf("Quick Mode: got %v >= %v", got, targetPods) + ctx.logf("Quick Mode: got %v >= %v", got, targetPods) return nil } if minPods < targetPods-1 { @@ -353,7 +359,7 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done } mes := fmt.Sprintf("got %v replicas, expected between [%v, %v] replicas for revision %s", got, targetPods-1, maxPods, ctx.resources.Revision.Name) - ctx.t.Log(mes) + ctx.logf(mes) if got < targetPods-1 || got > maxPods { return errors.New("final scale didn't fulfill constraints: " + mes) } @@ -371,7 +377,18 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done // from the given `done` channel will be sent within the `duration`. func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) { ctx.t.Helper() + wait := AutoscaleUpToNumPods(ctx, curPods, targetPods, done, quick) + if err := wait(); err != nil { + ctx.t.Fatal(err) + } +} +// AutoscaleUpToNumPods starts the traffic for AssertAutoscaleUpToNumPods and returns +// a function to wait for which will return any error from test execution. +// Starting the routines is separated from waiting for easy re-use in other +// places (e.g. upgrade tests). +func AutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) func() error { + ctx.t.Helper() // Relax the bounds to reduce the flakiness caused by sampling in the autoscaling algorithm. // Also adjust the values by the target utilization values. minPods := math.Floor(curPods/ctx.targetUtilization) - 1 @@ -393,7 +410,5 @@ func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, d return checkPodScale(ctx, targetPods, minPods, maxPods, done, quick) }) - if err := grp.Wait(); err != nil { - ctx.t.Fatal("Error: ", err) - } + return grp.Wait } diff --git a/test/e2e/autoscale_test.go b/test/e2e/autoscale_test.go index 387f7eb223db..0ee16e114e8e 100644 --- a/test/e2e/autoscale_test.go +++ b/test/e2e/autoscale_test.go @@ -41,12 +41,12 @@ func TestAutoscaleUpDownUp(t *testing.T) { t.Parallel() ctx := SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) AssertAutoscaleUpToNumPods(ctx, 1, 2, time.After(60*time.Second), true /* quick */) assertScaleDown(ctx) AssertAutoscaleUpToNumPods(ctx, 0, 2, time.After(60*time.Second), true /* quick */) } - func TestAutoscaleUpCountPods(t *testing.T) { t.Parallel() runAutoscaleUpCountPods(t, autoscaling.KPA, autoscaling.Concurrency) @@ -66,6 +66,7 @@ func runAutoscaleUpCountPods(t *testing.T, class, metric string) { } ctx := SetupSvc(t, class, metric, target, targetUtilization) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) ctx.t.Log("The autoscaler spins up additional replicas when traffic increases.") // Note: without the warm-up / gradual increase of load the test is @@ -89,6 +90,8 @@ func TestAutoscaleSustaining(t *testing.T) { t.Parallel() ctx := SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) + AssertAutoscaleUpToNumPods(ctx, 1, 10, time.After(2*time.Minute), false /* quick */) } @@ -105,6 +108,7 @@ func TestTargetBurstCapacity(t *testing.T) { autoscaling.TargetBurstCapacityKey: "7", autoscaling.PanicThresholdPercentageAnnotationKey: "200", // makes panicking rare })) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) cfg, err := autoscalerCM(ctx.clients) if err != nil { @@ -167,6 +171,7 @@ func TestTargetBurstCapacityMinusOne(t *testing.T) { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "-1", })) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) _, err := autoscalerCM(ctx.clients) if err != nil { @@ -193,6 +198,7 @@ func TestFastScaleToZero(t *testing.T) { autoscaling.TargetBurstCapacityKey: "-1", autoscaling.WindowAnnotationKey: autoscaling.WindowMin.String(), })) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) cfg, err := autoscalerCM(ctx.clients) if err != nil { diff --git a/test/e2e/grpc_test.go b/test/e2e/grpc_test.go index cc15ea03f424..b16242ac7944 100644 --- a/test/e2e/grpc_test.go +++ b/test/e2e/grpc_test.go @@ -324,15 +324,15 @@ func testGRPC(t *testing.T, f grpcTest, fopts ...rtesting.ServiceOption) { t.Log("Creating service for grpc-ping") - names := test.ResourceNames{ + names := &test.ResourceNames{ Service: test.ObjectNameForTest(t), Image: "grpc-ping", } fopts = append(fopts, rtesting.WithNamedPort("h2c")) - test.EnsureTearDown(t, clients, &names) - resources, err := v1test.CreateServiceReady(t, clients, &names, fopts...) + test.EnsureTearDown(t, clients, names) + resources, err := v1test.CreateServiceReady(t, clients, names, fopts...) if err != nil { t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err) } @@ -362,6 +362,7 @@ func testGRPC(t *testing.T, f grpcTest, fopts ...rtesting.ServiceOption) { f(&TestContext{ t: t, + logf: t.Logf, clients: clients, names: names, resources: resources, diff --git a/test/e2e/service_to_service_test.go b/test/e2e/service_to_service_test.go index 4e9bf49a0735..593f769eed02 100644 --- a/test/e2e/service_to_service_test.go +++ b/test/e2e/service_to_service_test.go @@ -243,6 +243,7 @@ func testSvcToSvcCallViaActivator(t *testing.T, clients *test.Clients, injectA b // Wait for the activator endpoints to equalize. if err := waitForActivatorEndpoints(&TestContext{ t: t, + logf: t.Logf, resources: resources, clients: clients, }); err != nil { diff --git a/test/e2e/websocket_test.go b/test/e2e/websocket_test.go index cf10defe0520..fa108efcf8d3 100644 --- a/test/e2e/websocket_test.go +++ b/test/e2e/websocket_test.go @@ -170,6 +170,7 @@ func TestWebSocketViaActivator(t *testing.T) { // Wait for the activator endpoints to equalize. if err := waitForActivatorEndpoints(&TestContext{ t: t, + logf: t.Logf, resources: resources, clients: clients, }); err != nil { diff --git a/test/ha/autoscaler_test.go b/test/ha/autoscaler_test.go index 43f52e0f7744..f548ef0bb0ec 100644 --- a/test/ha/autoscaler_test.go +++ b/test/ha/autoscaler_test.go @@ -55,7 +55,7 @@ func TestAutoscalerHA(t *testing.T) { resources := ctx.Resources() clients := ctx.Clients() - test.EnsureTearDown(t, clients, &names) + test.EnsureTearDown(t, clients, names) t.Log("Expected replicas = ", test.ServingFlags.Replicas) if err := pkgTest.WaitForDeploymentScale(context.Background(), clients.KubeClient, autoscalerDeploymentName, system.Namespace(), test.ServingFlags.Replicas); err != nil { @@ -100,7 +100,7 @@ func TestAutoscalerHA(t *testing.T) { } t.Log("Service should be able to generate a new revision after changing the leader controller") - names.Revision, err = v1test.WaitForServiceLatestRevision(clients, names) + names.Revision, err = v1test.WaitForServiceLatestRevision(clients, *names) if err != nil { t.Fatal("New image not reflected in Service:", err) } diff --git a/test/prober.go b/test/prober.go index 893453e56e00..de0702d71dc2 100644 --- a/test/prober.go +++ b/test/prober.go @@ -233,12 +233,19 @@ func RunRouteProber(logf logging.FormatLogger, clients *Clients, url *url.URL, o // against the default SLO, which requires perfect responses. // This takes `testing.T` so that it may be used in `defer`. func AssertProberDefault(t testing.TB, p Prober) { + t.Helper() + AssertProberSLO(t, p, 1.0) +} + +// AssertProberSLO is a helper for stopping the Prober and checking its SLI +// against the given SLO. +func AssertProberSLO(t testing.TB, p Prober, slo float64) { t.Helper() if err := p.Stop(); err != nil { t.Error("Stop()", "error", err.Error()) } // Default to 100% correct (typically used in conjunction with the low probe count above) - if err := CheckSLO(1.0, t.Name(), p); err != nil { + if err := CheckSLO(slo, t.Name(), p); err != nil { t.Error("CheckSLO()", "error", err.Error()) } } diff --git a/test/upgrade/README.md b/test/upgrade/README.md index 604f56c7f81f..d4d06302a414 100644 --- a/test/upgrade/README.md +++ b/test/upgrade/README.md @@ -25,40 +25,41 @@ At a high level, we want to do this: 1. Install knative at HEAD. 1. Test those resources, verify that we didn’t break anything. -To achieve that, we just have three separate build tags: +To achieve that, we utilize the [upgrade test framework](https://github.com/knative/pkg/tree/master/test/upgrade). +The framework runs tests in the following phases: 1. Install the latest release from GitHub. -1. Run the `preupgrade` tests in this directory. -1. Install at HEAD (`ko apply -Rf config/core/`). -1. Run the `postupgrade` tests in this directory. +1. Run the `PreUpgrade` tests. +1. Start the `Continual` tests. +1. Install from the HEAD of the master branch. +1. Run the `PostUpgrade` tests. 1. Install the latest release from GitHub. -1. Run the `postdowngrade` tests in this directory. +1. Run the `PostDowngrade` tests. +1. End the `Continual` tests and collect results. ## Tests -### Service test +#### PreUpgrade -This was stolen from the conformance tests but stripped down to check fewer -things. +Create a Service pointing to `image1`, ensure it responds correctly. -#### preupgrade - -Create a RunLatest Service pointing to `image1`, ensure it responds correctly. - -#### postupgrade +#### PostUpgrade Ensure the Service still responds correctly after upgrading. Update it to point -to `image2`, ensure it responds correctly. +to `image2`, ensure it responds correctly. Ensure that a new service +can be created after downgrade. -#### postdowngrade +#### PostDowngrade Ensure the Service still responds correctly after downgrading. Update it to -point back to `image1`, ensure it responds correctly. +point back to `image1`, ensure it responds correctly. Ensure that a new service +can be created after downgrade. -### Probe test +### Continual In order to verify that we don't have data-plane unavailability during our control-plane outages (when we're upgrading the knative/serving installation), we run a prober test that continually sends requests to a service during the entire upgrade process. When the upgrade completes, we make sure that none of -those requests failed. +those requests failed. We also run autoscale tests to ensure that autoscaling +works correctly during upgrades/downgrades. diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go new file mode 100644 index 000000000000..18e5f7d9e50d --- /dev/null +++ b/test/upgrade/autoscaler.go @@ -0,0 +1,91 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "time" + + pkgupgrade "knative.dev/pkg/test/upgrade" + "knative.dev/serving/pkg/apis/autoscaling" + rtesting "knative.dev/serving/pkg/testing/v1" + "knative.dev/serving/test" + "knative.dev/serving/test/e2e" +) + +const ( + containerConcurrency = 6 + targetUtilization = 0.7 + curPods = 1 + targetPods = 10 +) + +// AutoscaleSustainingTest checks that when traffic increases a knative app +// scales up and sustains the scale as long as the traffic sustains, despite whether +// it is switching modes between normal and panic. +func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { + var ctx *e2e.TestContext + var wait func() error + stopCh := make(chan time.Time) + return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingTest", + func(c pkgupgrade.Context) { + // Setup + ctx = e2e.SetupSvc(c.T, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, + rtesting.WithConfigAnnotations(map[string]string{ + autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. + })) + ctx.SetLogger(c.Log.Infof) + wait = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) + }, + func(c pkgupgrade.Context) { + test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) + // Verification is done inside e2e.AssertAutoscaleUpToNumPods. + // We're just giving it a signal. + close(stopCh) + if err := wait(); err != nil { + c.T.Error("Error: ", err) + } + }, + ) +} + +// AutoscaleSustainingWithTBCTest checks that when traffic increases and the activator is +// in the path a knative app scales up and sustains the scale. +func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { + var ctx *e2e.TestContext + var wait func() error + stopCh := make(chan time.Time) + return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingWithTBCTest", + func(c pkgupgrade.Context) { + // Setup + ctx = e2e.SetupSvc(c.T, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, + rtesting.WithConfigAnnotations(map[string]string{ + autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. + })) + ctx.SetLogger(c.Log.Infof) + wait = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) + }, + func(c pkgupgrade.Context) { + test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) + // Verification is done inside e2e.AssertAutoscaleUpToNumPods. + // We're just giving it a signal. + close(stopCh) + if err := wait(); err != nil { + c.T.Error("Error: ", err) + } + }, + ) +} diff --git a/test/upgrade/autoscaler_test.go b/test/upgrade/autoscaler_test.go deleted file mode 100644 index 533d3c8a1425..000000000000 --- a/test/upgrade/autoscaler_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// +build probe - -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "io/ioutil" - "testing" - "time" - - "knative.dev/serving/pkg/apis/autoscaling" - rtesting "knative.dev/serving/pkg/testing/v1" - "knative.dev/serving/test/e2e" -) - -const ( - containerConcurrency = 6 - targetUtilization = 0.7 - - autoscalingPipe = "/tmp/autoscaling-signal" - autoscalingTBCPipe = "/tmp/autoscaling-tbc-signal" -) - -// This test similar to TestAutoscaleSustaining in test/e2e/autoscale_test.go. It asserts -// the pods number is sustained during the whole cluster upgrade/downgrade process. -func TestAutoscaleSustaining(t *testing.T) { - t.Parallel() - // Create a named pipe and wait for the upgrade script to write to it - // to signal that we should stop testing. - createPipe(t, autoscalingPipe) - - ctx := e2e.SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, - rtesting.WithConfigAnnotations(map[string]string{ - autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. - })) - - stopCh := make(chan time.Time) - go func() { - // e2e-upgrade-test.sh will close this pipe to signal the upgrade is - // over, at which point we will finish the test. - ioutil.ReadFile(autoscalingPipe) - close(stopCh) - }() - - e2e.AssertAutoscaleUpToNumPods(ctx, 1, 10, stopCh, false /* quick */) -} - -func TestAutoscaleSustainingWithTBC(t *testing.T) { - t.Parallel() - // Create a named pipe and wait for the upgrade script to write to it - // to signal that we should stop testing. - createPipe(t, autoscalingTBCPipe) - - ctx := e2e.SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, - rtesting.WithConfigAnnotations(map[string]string{ - autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. - })) - - stopCh := make(chan time.Time) - go func() { - // e2e-upgrade-test.sh will close this pipe to signal the upgrade is - // over, at which point we will finish the test. - ioutil.ReadFile(autoscalingTBCPipe) - close(stopCh) - }() - - e2e.AssertAutoscaleUpToNumPods(ctx, 1, 10, stopCh, false /* quick */) -} diff --git a/test/upgrade/continual.go b/test/upgrade/continual.go new file mode 100644 index 000000000000..2df35d25ebd9 --- /dev/null +++ b/test/upgrade/continual.go @@ -0,0 +1,28 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import pkgupgrade "knative.dev/pkg/test/upgrade" + +// ContinualTests is an umbrella function for grouping all Serving continual/background tests. +func ContinualTests() []pkgupgrade.BackgroundOperation { + return []pkgupgrade.BackgroundOperation{ + ProbeTest(), + AutoscaleSustainingTest(), + AutoscaleSustainingWithTBCTest(), + } +} diff --git a/test/upgrade/installation/shell.go b/test/upgrade/installation/shell.go new file mode 100644 index 000000000000..d91716ecd8ad --- /dev/null +++ b/test/upgrade/installation/shell.go @@ -0,0 +1,59 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installation + +import ( + "knative.dev/hack/shell" + pkgupgrade "knative.dev/pkg/test/upgrade" +) + +// Head installs Knative Serving from the HEAD of the master branch. +func Head() pkgupgrade.Operation { + return install("ServingHead", "install_head_reuse_ingress") +} + +// LatestRelease installs Knative Serving from the latest stable release. +func LatestRelease() pkgupgrade.Operation { + return install("ServingLatestRelease", "install_latest_release") +} + +func install(installName, shellFunc string) pkgupgrade.Operation { + return pkgupgrade.NewOperation(installName, func(c pkgupgrade.Context) { + c.Log.Info("Running shell function: ", shellFunc) + if err := callShellFunction(shellFunc); err != nil { + c.T.Error(err) + } + }) +} + +func callShellFunction(funcName string) error { + loc, err := shell.NewProjectLocation("../../..") + if err != nil { + return err + } + exec := shell.NewExecutor(shell.ExecutorConfig{ + ProjectLocation: loc, + }) + fn := shell.Function{ + Script: shell.Script{ + Label: funcName, + ScriptPath: "test/e2e-common.sh", + }, + FunctionName: funcName, + } + return exec.RunFunction(fn) +} diff --git a/test/upgrade/service_postdowngrade_test.go b/test/upgrade/postdowngrade.go similarity index 73% rename from test/upgrade/service_postdowngrade_test.go rename to test/upgrade/postdowngrade.go index 26d1df3c5a31..61a8aee8e81f 100644 --- a/test/upgrade/service_postdowngrade_test.go +++ b/test/upgrade/postdowngrade.go @@ -1,7 +1,5 @@ -// +build postdowngrade - /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23,9 +21,9 @@ import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - _ "knative.dev/pkg/system/testing" ptest "knative.dev/pkg/test" + pkgupgrade "knative.dev/pkg/test/upgrade" serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names" rtesting "knative.dev/serving/pkg/testing/v1" "knative.dev/serving/test" @@ -33,7 +31,23 @@ import ( v1test "knative.dev/serving/test/v1" ) -func TestServicePostDowngrade(t *testing.T) { +// ServingPostDowngradeTests is an umbrella function for grouping all Serving post-downgrade tests. +func ServingPostDowngradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + ServicePostDowngradeTest(), + CreateNewServicePostDowngradeTest(), + } +} + +// ServicePostDowngradeTest verifies an existing service after downgrade. +func ServicePostDowngradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePostDowngradeTest", func(c pkgupgrade.Context) { + servicePostDowngrade(c.T) + }) +} + +func servicePostDowngrade(t *testing.T) { + t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -74,7 +88,9 @@ func TestServicePostDowngrade(t *testing.T) { assertServiceResourcesUpdated(t, clients, names, url.URL(), test.PizzaPlanetText1) } -func TestCreateNewServicePostDowngrade(t *testing.T) { - t.Parallel() - createNewService(postDowngradeServiceName, t) +// CreateNewServicePostDowngradeTest verifies creating a new service after downgrade. +func CreateNewServicePostDowngradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("CreateNewServicePostDowngradeTest", func(c pkgupgrade.Context) { + createNewService(postDowngradeServiceName, c.T) + }) } diff --git a/test/upgrade/service_postupgrade_test.go b/test/upgrade/postupgrade.go similarity index 71% rename from test/upgrade/service_postupgrade_test.go rename to test/upgrade/postupgrade.go index d26f8bfeb19b..711f2fd1dc22 100644 --- a/test/upgrade/service_postupgrade_test.go +++ b/test/upgrade/postupgrade.go @@ -1,7 +1,5 @@ -// +build postupgrade - /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25,6 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/pkg/ptr" ptest "knative.dev/pkg/test" + pkgupgrade "knative.dev/pkg/test/upgrade" v1 "knative.dev/serving/pkg/apis/serving/v1" serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names" rtesting "knative.dev/serving/pkg/testing/v1" @@ -33,9 +32,26 @@ import ( v1test "knative.dev/serving/test/v1" ) -func TestServicePostUpgrade(t *testing.T) { - t.Parallel() +// ServingPostUpgradeTests is an umbrella function for grouping all Serving post-upgrade tests. +func ServingPostUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + ServicePostUpgradeTest(), + ServicePostUpgradeFromScaleToZeroTest(), + BYORevisionPostUpgradeTest(), + CreateNewServicePostUpgradeTest(), + InitialScalePostUpgradeTest(), + } +} +// ServicePostUpgradeTest verifies an existing service after upgrade. +func ServicePostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePostUpgradeTest", func(c pkgupgrade.Context) { + servicePostUpgrade(c.T) + }) +} + +func servicePostUpgrade(t *testing.T) { + t.Parallel() clients := e2e.Setup(t) // Before updating the service, the route and configuration objects should @@ -70,14 +86,22 @@ func routeHasGeneration(clients *test.Clients, serviceName string, generation in return routeObj.Generation == int64(generation), nil } -func TestServicePostUpgradeFromScaleToZero(t *testing.T) { - t.Parallel() - updateService(scaleToZeroServiceName, t) +// ServicePostUpgradeFromScaleToZeroTest verifies a scaled-to-zero service after upgrade. +func ServicePostUpgradeFromScaleToZeroTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("PostUpgradeFromScaleToZeroTest", func(c pkgupgrade.Context) { + updateService(scaleToZeroServiceName, c.T) + }) } -// TestBYORevisionPostUpgrade attempts to update the RouteSpec of a Service using BYO Revision name. This +// BYORevisionPostUpgradeTest attempts to update the RouteSpec of a Service using BYO Revision name. This // test is meant to catch new defaults that break the immutability of BYO Revision name. -func TestBYORevisionPostUpgrade(t *testing.T) { +func BYORevisionPostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("BYORevisionPostUpgradeTest", func(c pkgupgrade.Context) { + bYORevisionPostUpgrade(c.T) + }) +} + +func bYORevisionPostUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -136,12 +160,22 @@ func updateService(serviceName string, t *testing.T) { assertServiceResourcesUpdated(t, clients, names, routeURL, test.PizzaPlanetText2) } -func TestCreateNewServicePostUpgrade(t *testing.T) { - t.Parallel() - createNewService(postUpgradeServiceName, t) +// CreateNewServicePostUpgradeTest verifies creating a new service after upgrade. +func CreateNewServicePostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("CreateNewServicePostUpgradeTest", func(c pkgupgrade.Context) { + createNewService(postUpgradeServiceName, c.T) + }) +} + +// InitialScalePostUpgradeTest verifies that the service is ready after upgrade +// despite the fact that it does not receive any requests. +func InitialScalePostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("InitialScalePostUpgradeTest", func(c pkgupgrade.Context) { + initialScalePostUpgrade(c.T) + }) } -func TestInitialScalePostUpgrade(t *testing.T) { +func initialScalePostUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) diff --git a/test/upgrade/service_preupgrade_test.go b/test/upgrade/preupgrade.go similarity index 62% rename from test/upgrade/service_preupgrade_test.go rename to test/upgrade/preupgrade.go index 03516d8cc9d4..77fb99dfbb1f 100644 --- a/test/upgrade/service_preupgrade_test.go +++ b/test/upgrade/preupgrade.go @@ -1,7 +1,5 @@ -// +build preupgrade - /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,6 +19,7 @@ package upgrade import ( "testing" + pkgupgrade "knative.dev/pkg/test/upgrade" "knative.dev/serving/pkg/apis/autoscaling" revisionresourcenames "knative.dev/serving/pkg/reconciler/revision/resources/names" rtesting "knative.dev/serving/pkg/testing/v1" @@ -29,7 +28,24 @@ import ( v1test "knative.dev/serving/test/v1" ) -func TestServicePreUpgrade(t *testing.T) { +// ServingPreUpgradeTests is an umbrella function for grouping all Serving pre-upgrade tests. +func ServingPreUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + ServicePreUpgradeTest(), + ServicePreUpgradeAndScaleToZeroTest(), + BYORevisionPreUpgradeTest(), + InitialScalePreUpgradeTest(), + } +} + +// ServicePreUpgradeTest creates a service before upgrade. +func ServicePreUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePreUpgradeTest", func(c pkgupgrade.Context) { + servicePreUpgrade(c.T) + }) +} + +func servicePreUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) @@ -50,7 +66,15 @@ func TestServicePreUpgrade(t *testing.T) { assertServiceResourcesUpdated(t, clients, names, url, test.PizzaPlanetText1) } -func TestServicePreUpgradeAndScaleToZero(t *testing.T) { +// ServicePreUpgradeAndScaleToZeroTest creates a new service before +// upgrade and wait for it to scale to zero. +func ServicePreUpgradeAndScaleToZeroTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePreUpgradeAndScaleToZeroTest", func(c pkgupgrade.Context) { + servicePreUpgradeAndScaleToZero(c.T) + }) +} + +func servicePreUpgradeAndScaleToZero(t *testing.T) { t.Parallel() clients := e2e.Setup(t) @@ -75,9 +99,15 @@ func TestServicePreUpgradeAndScaleToZero(t *testing.T) { } } -// TestBYORevisionUpgrade creates a Service that uses the BYO Revision name functionality. This test +// BYORevisionPreUpgradeTest creates a Service that uses the BYO Revision name functionality. This test // is meant to catch new defaults that break bring your own revision name immutability. -func TestBYORevisionPreUpgrade(t *testing.T) { +func BYORevisionPreUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("BYORevisionPreUpgradeTest", func(c pkgupgrade.Context) { + bYORevisionPreUpgrade(c.T) + }) +} + +func bYORevisionPreUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -91,7 +121,15 @@ func TestBYORevisionPreUpgrade(t *testing.T) { } } -func TestInitialScalePreUpgrade(t *testing.T) { +// InitialScalePreUpgradeTest creates a service and lets it scale down to zero without +// sending any requests to it. +func InitialScalePreUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("InitialScalePreUpgradeTest", func(c pkgupgrade.Context) { + initialScalePreUpgrade(c.T) + }) +} + +func initialScalePreUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ diff --git a/test/upgrade/probe.go b/test/upgrade/probe.go new file mode 100644 index 000000000000..38c4ebc31719 --- /dev/null +++ b/test/upgrade/probe.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "flag" + + pkgupgrade "knative.dev/pkg/test/upgrade" + "knative.dev/serving/test" + "knative.dev/serving/test/e2e" + v1test "knative.dev/serving/test/v1" +) + +var successFraction = flag.Float64("probe.success_fraction", 1.0, "Fraction of probes required to pass during upgrade.") + +// ProbeTest sends requests to Knative services while performing an upgrade +// and verifies the expected success rate. +func ProbeTest() pkgupgrade.BackgroundOperation { + var clients *test.Clients + var names *test.ResourceNames + var prober test.Prober + return pkgupgrade.NewBackgroundVerification("ProbeTest", + func(c pkgupgrade.Context) { + // Setup + clients = e2e.Setup(c.T) + names = &test.ResourceNames{ + Service: "upgrade-probe", + Image: test.PizzaPlanet1, + } + objects, err := v1test.CreateServiceReady(c.T, clients, names) + if err != nil { + c.T.Fatal("Failed to create Service:", err) + } + url := objects.Service.Status.URL.URL() + + // This polls until we get a 200 with the right body. + assertServiceResourcesUpdated(c.T, clients, *names, url, test.PizzaPlanetText1) + + prober = test.RunRouteProber(c.T.Logf, clients, url, test.AddRootCAtoTransport(context.Background(), c.T.Logf, clients, test.ServingFlags.HTTPS)) + }, + func(c pkgupgrade.Context) { + // Verify + test.EnsureTearDown(c.T, clients, names) + test.AssertProberSLO(c.T, prober, *successFraction) + }, + ) +} diff --git a/test/upgrade/probe_test.go b/test/upgrade/probe_test.go deleted file mode 100644 index 981ee29579e7..000000000000 --- a/test/upgrade/probe_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// +build probe - -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "context" - "flag" - "io/ioutil" - "testing" - - "knative.dev/serving/test" - "knative.dev/serving/test/e2e" - v1test "knative.dev/serving/test/v1" -) - -const probePipe = "/tmp/prober-signal" - -var successFraction = flag.Float64("probe.success_fraction", 1.0, "Fraction of probes required to pass during upgrade.") - -func TestProbe(t *testing.T) { - t.Parallel() - // We run the prober as a golang test because it fits in nicely with - // the rest of our integration tests, and AssertProberDefault needs - // a *testing.T. Unfortunately, "go test" intercepts signals, so we - // can't coordinate with the test by just sending e.g. SIGCONT, so we - // create a named pipe and wait for the upgrade script to write to it - // to signal that we should stop probing. - createPipe(t, probePipe) - - clients := e2e.Setup(t) - names := test.ResourceNames{ - Service: "upgrade-probe", - Image: test.PizzaPlanet1, - } - - test.EnsureTearDown(t, clients, &names) - - objects, err := v1test.CreateServiceReady(t, clients, &names) - if err != nil { - t.Fatal("Failed to create Service:", err) - } - url := objects.Service.Status.URL.URL() - - // This polls until we get a 200 with the right body. - assertServiceResourcesUpdated(t, clients, names, url, test.PizzaPlanetText1) - - prober := test.RunRouteProber(t.Logf, clients, url, test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)) - defer test.CheckSLO(*successFraction, t.Name(), prober) - - // e2e-upgrade-test.sh will close this pipe to signal the upgrade is - // over, at which point we will finish the test and check the prober. - ioutil.ReadFile(probePipe) -} diff --git a/test/upgrade/upgrade.go b/test/upgrade/upgrade.go index edd135a3c63c..a5fcfaa98e41 100644 --- a/test/upgrade/upgrade.go +++ b/test/upgrade/upgrade.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,11 +18,8 @@ package upgrade import ( "context" - "errors" "fmt" "net/url" - "os" - "syscall" "testing" // Mysteriously required to support GCP auth (required by k8s libs). @@ -66,6 +63,7 @@ func assertServiceResourcesUpdated(t testing.TB, clients *test.Clients, names te } func createNewService(serviceName string, t *testing.T) { + t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -80,17 +78,3 @@ func createNewService(serviceName string, t *testing.T) { url := resources.Service.Status.URL.URL() assertServiceResourcesUpdated(t, clients, names, url, test.PizzaPlanetText1) } - -// createPipe create a named pipe. It fails the test if any error except -// already exist happens. -func createPipe(t *testing.T, name string) { - if err := syscall.Mkfifo(name, 0666); err != nil { - if !errors.Is(err, os.ErrExist) { - t.Fatal("Failed to create pipe:", err) - } - } - - test.EnsureCleanup(t, func() { - os.Remove(name) - }) -} diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go new file mode 100644 index 000000000000..658bd3a5626a --- /dev/null +++ b/test/upgrade/upgrade_test.go @@ -0,0 +1,61 @@ +// +build upgrade + +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" + + "go.uber.org/zap" + _ "knative.dev/pkg/system/testing" + pkgupgrade "knative.dev/pkg/test/upgrade" + "knative.dev/serving/test/upgrade/installation" +) + +func TestServingUpgrades(t *testing.T) { + c := newUpgradeConfig(t) + suite := pkgupgrade.Suite{ + Tests: pkgupgrade.Tests{ + PreUpgrade: ServingPreUpgradeTests(), + PostUpgrade: ServingPostUpgradeTests(), + PostDowngrade: ServingPostDowngradeTests(), + Continual: ContinualTests(), + }, + Installations: pkgupgrade.Installations{ + Base: []pkgupgrade.Operation{ + // Do nothing. The initial version is already installed by scripts + // together with additional test resources. + }, + UpgradeWith: []pkgupgrade.Operation{ + installation.Head(), + }, + DowngradeWith: []pkgupgrade.Operation{ + installation.LatestRelease(), + }, + }, + } + suite.Execute(c) +} + +func newUpgradeConfig(t *testing.T) pkgupgrade.Configuration { + log, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + return pkgupgrade.Configuration{T: t, Log: log} +} diff --git a/third_party/VENDOR-LICENSE/knative.dev/hack/shell/LICENSE b/third_party/VENDOR-LICENSE/knative.dev/hack/shell/LICENSE new file mode 100644 index 000000000000..261eeb9e9f8b --- /dev/null +++ b/third_party/VENDOR-LICENSE/knative.dev/hack/shell/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/knative.dev/hack/shell/executor.go b/vendor/knative.dev/hack/shell/executor.go new file mode 100644 index 000000000000..e6308a0706fd --- /dev/null +++ b/vendor/knative.dev/hack/shell/executor.go @@ -0,0 +1,189 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "time" +) + +const ( + defaultLabelOut = "[OUT]" + defaultLabelErr = "[ERR]" + executeMode = 0700 +) + +// ErrNoProjectLocation is returned if user didnt provided the project location. +var ErrNoProjectLocation = errors.New("project location isn't provided") + +// NewExecutor creates a new executor from given config. +func NewExecutor(config ExecutorConfig) Executor { + configureDefaultValues(&config) + return &streamingExecutor{ + ExecutorConfig: config, + } +} + +// RunScript executes a shell script with args. +func (s *streamingExecutor) RunScript(script Script, args ...string) error { + err := validate(s.ExecutorConfig) + if err != nil { + return err + } + cnt := script.scriptContent(s.ProjectLocation, args) + return withTempScript(cnt, func(bin string) error { + return stream(bin, s.ExecutorConfig, script.Label) + }) +} + +// RunFunction executes a shell function with args. +func (s *streamingExecutor) RunFunction(fn Function, args ...string) error { + err := validate(s.ExecutorConfig) + if err != nil { + return err + } + cnt := fn.scriptContent(s.ProjectLocation, args) + return withTempScript(cnt, func(bin string) error { + return stream(bin, s.ExecutorConfig, fn.Label) + }) +} + +type streamingExecutor struct { + ExecutorConfig +} + +func validate(config ExecutorConfig) error { + if config.ProjectLocation == nil { + return ErrNoProjectLocation + } + return nil +} + +func configureDefaultValues(config *ExecutorConfig) { + if config.Out == nil { + config.Out = os.Stdout + } + if config.Err == nil { + config.Err = os.Stderr + } + if config.LabelOut == "" { + config.LabelOut = defaultLabelOut + } + if config.LabelErr == "" { + config.LabelErr = defaultLabelErr + } + if config.Environ == nil { + config.Environ = os.Environ() + } + if !config.SkipDate && config.DateFormat == "" { + config.DateFormat = time.StampMilli + } + if config.PrefixFunc == nil { + config.PrefixFunc = defaultPrefixFunc + } +} + +func stream(bin string, cfg ExecutorConfig, label string) error { + c := exec.Command(bin) + c.Env = cfg.Environ + c.Stdout = NewPrefixer(cfg.Out, prefixFunc(StreamTypeOut, label, cfg)) + c.Stderr = NewPrefixer(cfg.Err, prefixFunc(StreamTypeErr, label, cfg)) + return c.Run() +} + +func prefixFunc(st StreamType, label string, cfg ExecutorConfig) func() string { + return func() string { + return cfg.PrefixFunc(st, label, cfg) + } +} + +func defaultPrefixFunc(st StreamType, label string, cfg ExecutorConfig) string { + sep := " " + var buf []string + if !cfg.SkipDate { + dt := time.Now().Format(cfg.DateFormat) + buf = append(buf, dt) + } + buf = append(buf, label) + switch st { + case StreamTypeOut: + buf = append(buf, cfg.LabelOut) + case StreamTypeErr: + buf = append(buf, cfg.LabelErr) + } + return strings.Join(buf, sep) + sep +} + +func withTempScript(contents string, fn func(bin string) error) error { + tmpfile, err := ioutil.TempFile("", "shellout-*.sh") + if err != nil { + return err + } + _, err = tmpfile.WriteString(contents) + if err != nil { + return err + } + err = tmpfile.Chmod(executeMode) + if err != nil { + return err + } + err = tmpfile.Close() + if err != nil { + return err + } + defer func() { + // clean up + _ = os.Remove(tmpfile.Name()) + }() + + return fn(tmpfile.Name()) +} + +func (fn *Function) scriptContent(location ProjectLocation, args []string) string { + return fmt.Sprintf(`#!/usr/bin/env bash + +set -Eeuo pipefail + +cd "%s" +source %s + +%s %s +`, location.RootPath(), fn.ScriptPath, fn.FunctionName, quoteArgs(args)) +} + +func (sc *Script) scriptContent(location ProjectLocation, args []string) string { + return fmt.Sprintf(`#!/usr/bin/env bash + +set -Eeuo pipefail + +cd "%s" +%s %s +`, location.RootPath(), sc.ScriptPath, quoteArgs(args)) +} + +func quoteArgs(args []string) string { + quoted := make([]string, len(args)) + for i, arg := range args { + quoted[i] = "\"" + strings.ReplaceAll(arg, "\"", "\\\"") + "\"" + } + return strings.Join(quoted, " ") +} diff --git a/vendor/knative.dev/hack/shell/fail-example.sh b/vendor/knative.dev/hack/shell/fail-example.sh new file mode 100644 index 000000000000..551ce662a340 --- /dev/null +++ b/vendor/knative.dev/hack/shell/fail-example.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "$*" >&2 diff --git a/vendor/knative.dev/hack/shell/prefixer.go b/vendor/knative.dev/hack/shell/prefixer.go new file mode 100644 index 000000000000..273ee1cc4fff --- /dev/null +++ b/vendor/knative.dev/hack/shell/prefixer.go @@ -0,0 +1,68 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "bytes" + "io" +) + +// NewPrefixer creates a new prefixer that forwards all calls to Write() to +// writer.Write() with all lines prefixed with the value of prefix. Having a +// function instead of a static prefix allows to print timestamps or other +// changing information. +func NewPrefixer(writer io.Writer, prefix func() string) io.Writer { + return &prefixer{prefix: prefix, writer: writer, trailingNewline: true} +} + +type prefixer struct { + prefix func() string + writer io.Writer + trailingNewline bool + buf bytes.Buffer // reuse buffer to save allocations +} + +func (pf *prefixer) Write(payload []byte) (int, error) { + pf.buf.Reset() // clear the buffer + + for _, b := range payload { + if pf.trailingNewline { + pf.buf.WriteString(pf.prefix()) + pf.trailingNewline = false + } + + pf.buf.WriteByte(b) + + if b == '\n' { + // do not print the prefix right after the newline character as this might + // be the very last character of the stream and we want to avoid a trailing prefix. + pf.trailingNewline = true + } + } + + n, err := pf.writer.Write(pf.buf.Bytes()) + if err != nil { + // never return more than original length to satisfy io.Writer interface + if n > len(payload) { + n = len(payload) + } + return n, err + } + + // return original length to satisfy io.Writer interface + return len(payload), nil +} diff --git a/vendor/knative.dev/hack/shell/project.go b/vendor/knative.dev/hack/shell/project.go new file mode 100644 index 000000000000..f4cd1681adc0 --- /dev/null +++ b/vendor/knative.dev/hack/shell/project.go @@ -0,0 +1,80 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "errors" + "fmt" + "path" + "regexp" + "runtime" +) + +var ( + // ErrCantGetCaller is raised when we can't calculate a caller of NewProjectLocation. + ErrCantGetCaller = errors.New("can't get caller") + + // ErrCallerNotAllowed is raised when user tries to use this shell-out package + // outside of allowed places. This package is deprecated from start and was + // introduced to allow rewriting of shell code to Golang in small chunks. + ErrCallerNotAllowed = errors.New("don't try use knative.dev/hack/shell package outside of allowed places") +) + +// NewProjectLocation creates a ProjectLocation that is used to calculate +// relative paths within the project. +func NewProjectLocation(pathToRoot string) (ProjectLocation, error) { + _, filename, _, ok := runtime.Caller(1) + if !ok { + return nil, ErrCantGetCaller + } + err := ensureIsValid(filename) + if err != nil { + return nil, err + } + return &callerLocation{ + caller: filename, + pathToRoot: pathToRoot, + }, nil +} + +// RootPath return a path to root of the project. +func (c *callerLocation) RootPath() string { + return path.Join(path.Dir(c.caller), c.pathToRoot) +} + +// callerLocation holds a caller Go file, and a relative location to a project +// root directory. This information can be used to calculate relative paths and +// properly source shell scripts. +type callerLocation struct { + caller string + pathToRoot string +} + +func ensureIsValid(filename string) error { + validPaths := []string{ + "knative.+/test/upgrade/", + "knative(:?\\.dev/|-)hack/shell/", + } + for _, validPath := range validPaths { + r := regexp.MustCompile(validPath) + if loc := r.FindStringIndex(filename); loc != nil { + return nil + } + } + return fmt.Errorf("%w, tried using from: %s", + ErrCallerNotAllowed, filename) +} diff --git a/vendor/knative.dev/hack/shell/types.go b/vendor/knative.dev/hack/shell/types.go new file mode 100644 index 000000000000..8e34515206dd --- /dev/null +++ b/vendor/knative.dev/hack/shell/types.go @@ -0,0 +1,82 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import "io" + +// ProjectLocation represents a project location on a file system. +type ProjectLocation interface { + RootPath() string +} + +// Script represents a script to be executed. +type Script struct { + Label string + ScriptPath string +} + +// Function represents a function, whom will be sourced from Script file, +// and executed. +type Function struct { + Script + FunctionName string +} + +// ExecutorConfig holds a executor configuration options. +type ExecutorConfig struct { + ProjectLocation + Streams + Labels + Environ []string +} + +// StreamType represets either output or error stream. +type StreamType int + +const ( + // StreamTypeOut represents process output stream. + StreamTypeOut StreamType = iota + // StreamTypeErr represents process error stream. + StreamTypeErr +) + +// PrefixFunc is used to build a prefix that will be added to each line of the +// script/function output or error stream. +type PrefixFunc func(st StreamType, label string, config ExecutorConfig) string + +// Labels holds a labels to be used to prefix Out and Err streams of executed +// shells scripts/functions. +type Labels struct { + LabelOut string + LabelErr string + SkipDate bool + DateFormat string + PrefixFunc +} + +// Streams holds a streams of a shell scripts/functions. +type Streams struct { + Out io.Writer + Err io.Writer +} + +// Executor represents a executor that can execute shell scripts and call +// functions directly. +type Executor interface { + RunScript(script Script, args ...string) error + RunFunction(fn Function, args ...string) error +} diff --git a/vendor/knative.dev/pkg/test/upgrade/functions.go b/vendor/knative.dev/pkg/test/upgrade/functions.go new file mode 100644 index 000000000000..07cd80e92ac5 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/functions.go @@ -0,0 +1,162 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "time" + + "go.uber.org/zap" +) + +// Execute the Suite of upgrade tests with a Configuration given. +func (s *Suite) Execute(c Configuration) { + l := c.logger() + se := suiteExecution{ + suite: enrichSuite(s), + configuration: c, + failed: false, + logger: l, + } + l.Info("🏃 Running upgrade test suite...") + + se.execute() + + if !se.failed { + l.Info("🥳🎉 Success! Upgrade suite completed without errors.") + } else { + l.Error("💣🤬💔️ Upgrade suite have failed!") + } +} + +// NewOperation creates a new upgrade operation or test. +func NewOperation(name string, handler func(c Context)) Operation { + return &simpleOperation{name: name, handler: handler} +} + +// NewBackgroundVerification is convenience function to easily setup a +// background operation that will setup environment and then verify environment +// status after receiving a StopEvent. +func NewBackgroundVerification(name string, setup func(c Context), verify func(c Context)) BackgroundOperation { + return NewBackgroundOperation(name, setup, func(bc BackgroundContext) { + WaitForStopEvent(bc, WaitForStopEventConfiguration{ + Name: name, + OnStop: func(event StopEvent) { + verify(Context{ + T: event.T, + Log: bc.Log, + }) + }, + OnWait: DefaultOnWait, + WaitTime: DefaultWaitTime, + }) + }) +} + +// NewBackgroundOperation creates a new background operation or test that can be +// notified to stop its operation. +func NewBackgroundOperation(name string, setup func(c Context), + handler func(bc BackgroundContext)) BackgroundOperation { + return &simpleBackgroundOperation{ + name: name, + setup: setup, + handler: handler, + } +} + +// WaitForStopEvent will wait until upgrade suite sends a stop event to it. +// After that happen a handler is invoked to verify environment state and report +// failures. +func WaitForStopEvent(bc BackgroundContext, w WaitForStopEventConfiguration) { + for { + select { + case stopEvent := <-bc.Stop: + handleStopEvent(stopEvent, bc, w) + return + default: + w.OnWait(bc, w) + } + time.Sleep(w.WaitTime) + } +} + +func (c Configuration) logger() *zap.SugaredLogger { + return c.Log.Sugar() +} + +// Name returns a friendly human readable text. +func (s *StopEvent) Name() string { + return s.name +} + +func handleStopEvent( + se StopEvent, + bc BackgroundContext, + wc WaitForStopEventConfiguration, +) { + bc.Log.Infof("%s have received a stop event: %s", wc.Name, se.Name()) + defer close(se.Finished) + wc.OnStop(se) +} + +func enrichSuite(s *Suite) *enrichedSuite { + es := &enrichedSuite{ + installations: s.Installations, + tests: enrichedTests{ + preUpgrade: s.Tests.PreUpgrade, + postUpgrade: s.Tests.PostUpgrade, + postDowngrade: s.Tests.PostDowngrade, + continual: make([]stoppableOperation, len(s.Tests.Continual)), + }, + } + for i, test := range s.Tests.Continual { + es.tests.continual[i] = stoppableOperation{ + BackgroundOperation: test, + stop: make(chan StopEvent), + } + } + return es +} + +// Name is a human readable operation title, and it will be used in t.Run. +func (h *simpleOperation) Name() string { + return h.name +} + +// Handler is a function that will be called to perform an operation. +func (h *simpleOperation) Handler() func(c Context) { + return h.handler +} + +// Name is a human readable operation title, and it will be used in t.Run. +func (s *simpleBackgroundOperation) Name() string { + return s.name +} + +// Setup method may be used to set up environment before upgrade/downgrade is +// performed. +func (s *simpleBackgroundOperation) Setup() func(c Context) { + return s.setup +} + +// Handler will be executed in background while upgrade/downgrade is being +// executed. It can be used to constantly validate environment during that +// time and/or wait for StopEvent being sent. After StopEvent is received +// user should validate environment, clean up resources, and report found +// issues to testing.T forwarded in StepEvent. +func (s *simpleBackgroundOperation) Handler() func(bc BackgroundContext) { + return s.handler +} diff --git a/vendor/knative.dev/pkg/test/upgrade/private_types.go b/vendor/knative.dev/pkg/test/upgrade/private_types.go new file mode 100644 index 000000000000..ab930e41bf62 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/private_types.go @@ -0,0 +1,63 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import "go.uber.org/zap" + +type suiteExecution struct { + suite *enrichedSuite + configuration Configuration + failed bool + logger *zap.SugaredLogger +} + +type enrichedSuite struct { + installations Installations + tests enrichedTests +} + +type enrichedTests struct { + preUpgrade []Operation + postUpgrade []Operation + postDowngrade []Operation + continual []stoppableOperation +} + +type stoppableOperation struct { + BackgroundOperation + stop chan StopEvent +} + +type operationGroup struct { + num int + operations []Operation + groupName string + groupTemplate string + elementTemplate string + skippingGroupTemplate string +} + +type simpleOperation struct { + name string + handler func(c Context) +} + +type simpleBackgroundOperation struct { + name string + setup func(c Context) + handler func(bc BackgroundContext) +} diff --git a/vendor/knative.dev/pkg/test/upgrade/steps.go b/vendor/knative.dev/pkg/test/upgrade/steps.go new file mode 100644 index 000000000000..1c08038ec74e --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/steps.go @@ -0,0 +1,155 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" +) + +const skippingOperationTemplate = `Skipping "%s" as previous operation have failed` + +func (se *suiteExecution) installingBase(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.Base, + groupName: "InstallingBase", + elementTemplate: `%d.%d) Installing base install of "%s".`, + skippingGroupTemplate: "%d) 💿 No base installation registered. Skipping.", + groupTemplate: "%d) 💿 Installing base installations. %d are registered.", + }) +} + +func (se *suiteExecution) preUpgradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.preUpgrade, + groupName: "PreUpgradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No pre upgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality before upgrade is performed." + + " %d tests are registered.", + }) +} + +func (se *suiteExecution) startContinualTests(num int) { + l := se.logger + operations := se.suite.tests.continual + groupTemplate := "%d) 🔄 Starting continual tests. " + + "%d tests are registered." + elementTemplate := `%d.%d) Starting continual tests of "%s".` + numOps := len(operations) + se.configuration.T.Run("ContinualTests", func(t *testing.T) { + if numOps > 0 { + l.Infof(groupTemplate, num, numOps) + for i := range operations { + operation := operations[i] + l.Infof(elementTemplate, num, i+1, operation.Name()) + if se.failed { + l.Debugf(skippingOperationTemplate, operation.Name()) + return + } + setup := operation.Setup() + t.Run("Setup"+operation.Name(), func(t *testing.T) { + setup(Context{T: t, Log: l}) + }) + handler := operation.Handler() + go func() { + bc := BackgroundContext{Log: l, Stop: operation.stop} + handler(bc) + }() + + se.failed = se.failed || t.Failed() + if se.failed { + return + } + } + + } else { + l.Infof("%d) 🔄 No continual tests registered. Skipping.", num) + } + }) +} + +func (se *suiteExecution) verifyContinualTests(num int) { + l := se.logger + testsCount := len(se.suite.tests.continual) + if testsCount > 0 { + se.configuration.T.Run("VerifyContinualTests", func(t *testing.T) { + l.Infof("%d) ✋ Verifying %d running continual tests.", num, testsCount) + for i, operation := range se.suite.tests.continual { + t.Run(operation.Name(), func(t *testing.T) { + l.Infof(`%d.%d) Verifying "%s".`, num, i+1, operation.Name()) + finished := make(chan struct{}) + operation.stop <- StopEvent{ + T: t, + Finished: finished, + name: "Stop of " + operation.Name(), + } + <-finished + se.failed = se.failed || t.Failed() + l.Debugf(`Finished "%s"`, operation.Name()) + }) + } + }) + } +} + +func (se *suiteExecution) upgradeWith(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.UpgradeWith, + groupName: "UpgradeWith", + elementTemplate: `%d.%d) Upgrading with "%s".`, + skippingGroupTemplate: "%d) 📀 No upgrade operations registered. Skipping.", + groupTemplate: "%d) 📀 Upgrading with %d registered operations.", + }) +} + +func (se *suiteExecution) postUpgradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.postUpgrade, + groupName: "PostUpgradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No post upgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality after upgrade is performed." + + " %d tests are registered.", + }) +} + +func (se *suiteExecution) downgradeWith(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.DowngradeWith, + groupName: "DowngradeWith", + elementTemplate: `%d.%d) Downgrading with "%s".`, + skippingGroupTemplate: "%d) 💿 No downgrade operations registered. Skipping.", + groupTemplate: "%d) 💿 Downgrading with %d registered operations.", + }) +} + +func (se *suiteExecution) postDowngradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.postDowngrade, + groupName: "PostDowngradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No post downgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality after downgrade is performed." + + " %d tests are registered.", + }) +} diff --git a/vendor/knative.dev/pkg/test/upgrade/suite_execution.go b/vendor/knative.dev/pkg/test/upgrade/suite_execution.go new file mode 100644 index 000000000000..d926d4076187 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/suite_execution.go @@ -0,0 +1,85 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" +) + +func (se *suiteExecution) processOperationGroup(op operationGroup) { + l := se.logger + se.configuration.T.Run(op.groupName, func(t *testing.T) { + if len(op.operations) > 0 { + l.Infof(op.groupTemplate, op.num, len(op.operations)) + for i, operation := range op.operations { + l.Infof(op.elementTemplate, op.num, i+1, operation.Name()) + if se.failed { + l.Debugf(skippingOperationTemplate, operation.Name()) + return + } + handler := operation.Handler() + t.Run(operation.Name(), func(t *testing.T) { + handler(Context{T: t, Log: l}) + }) + se.failed = se.failed || t.Failed() + if se.failed { + return + } + } + } else { + l.Infof(op.skippingGroupTemplate, op.num) + } + }) +} + +func (se *suiteExecution) execute() { + idx := 1 + operations := []func(num int){ + se.installingBase, + se.preUpgradeTests, + } + for _, operation := range operations { + operation(idx) + idx++ + if se.failed { + return + } + } + + se.startContinualTests(idx) + idx++ + if se.failed { + return + } + defer func() { + se.verifyContinualTests(idx) + }() + + operations = []func(num int){ + se.upgradeWith, + se.postUpgradeTests, + se.downgradeWith, + se.postDowngradeTests, + } + for _, operation := range operations { + operation(idx) + idx++ + if se.failed { + return + } + } +} diff --git a/vendor/knative.dev/pkg/test/upgrade/types.go b/vendor/knative.dev/pkg/test/upgrade/types.go new file mode 100644 index 000000000000..e4324f4a0c8f --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/types.go @@ -0,0 +1,123 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" + "time" + + "go.uber.org/zap" +) + +// Suite represents a upgrade tests suite that can be executed and will perform +// execution in predictable manner. +type Suite struct { + Tests Tests + Installations Installations +} + +// Tests holds a list of operations for various part of upgrade suite. +type Tests struct { + PreUpgrade []Operation + PostUpgrade []Operation + PostDowngrade []Operation + Continual []BackgroundOperation +} + +// Installations holds a list of operations that will install Knative components +// in different versions. +type Installations struct { + Base []Operation + UpgradeWith []Operation + DowngradeWith []Operation +} + +// Operation represents a upgrade test operation like test or installation that +// can be provided by specific component or reused in aggregating components. +type Operation interface { + // Name is a human readable operation title, and it will be used in t.Run. + Name() string + // Handler is a function that will be called to perform an operation. + Handler() func(c Context) +} + +// BackgroundOperation represents a upgrade test operation that will be +// performed in background while other operations is running. To achieve that +// a passed BackgroundContext should be used to synchronize it's operations with +// Ready and Stop channels. +type BackgroundOperation interface { + // Name is a human readable operation title, and it will be used in t.Run. + Name() string + // Setup method may be used to set up environment before upgrade/downgrade is + // performed. + Setup() func(c Context) + // Handler will be executed in background while upgrade/downgrade is being + // executed. It can be used to constantly validate environment during that + // time and/or wait for StopEvent being sent. After StopEvent is received + // user should validate environment, clean up resources, and report found + // issues to testing.T forwarded in StepEvent. + Handler() func(bc BackgroundContext) +} + +// Context is an object that is passed to every operation. It contains testing.T +// for error reporting and zap.SugaredLogger for unbuffered logging. +type Context struct { + T *testing.T + Log *zap.SugaredLogger +} + +// BackgroundContext is a upgrade test execution context that will be passed +// down to each handler of BackgroundOperation. It contains a StopEvent channel +// which end user should use to obtain a testing.T for error reporting. Until +// StopEvent is sent user may use zap.SugaredLogger to log state of execution if +// necessary. +type BackgroundContext struct { + Log *zap.SugaredLogger + Stop <-chan StopEvent +} + +// StopEvent represents an event that is to be received by background operation +// to indicate that is should stop it's operations and validate results using +// passed T. User should use Finished channel to signalize upgrade suite that +// all stop & verify operations are finished and it is safe to end tests. +type StopEvent struct { + T *testing.T + Finished chan<- struct{} + name string +} + +// WaitForStopEventConfiguration holds a values to be used be WaitForStopEvent +// function. OnStop will be called when StopEvent is sent. OnWait will be +// invoked in a loop while waiting, and each wait act is driven by WaitTime +// amount. +type WaitForStopEventConfiguration struct { + Name string + OnStop func(event StopEvent) + OnWait func(bc BackgroundContext, self WaitForStopEventConfiguration) + WaitTime time.Duration +} + +// Configuration holds required and optional configuration to run upgrade tests. +type Configuration struct { + T *testing.T + Log *zap.Logger +} + +// SuiteExecutor is to execute upgrade test suite. +type SuiteExecutor interface { + Execute(c Configuration) +} diff --git a/vendor/knative.dev/pkg/test/upgrade/vars.go b/vendor/knative.dev/pkg/test/upgrade/vars.go new file mode 100644 index 000000000000..6a9527b35748 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/vars.go @@ -0,0 +1,32 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import "time" + +var ( + // DefaultWaitTime holds a default value for WaitForStopEventConfiguration + // when used within a NewBackgroundVerification function. + DefaultWaitTime = 20 * time.Millisecond + + // DefaultOnWait is a implementation that will be called by default for each + // wait performed by WaitForStopEvent when used within + // NewBackgroundVerification function. + DefaultOnWait = func(bc BackgroundContext, self WaitForStopEventConfiguration) { + // do nothing by default + } +) diff --git a/vendor/modules.txt b/vendor/modules.txt index a47bb01de1b8..5ac625083816 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -987,6 +987,7 @@ knative.dev/caching/pkg/client/listers/caching/v1alpha1 # knative.dev/hack v0.0.0-20201214230143-4ed1ecb8db24 ## explicit knative.dev/hack +knative.dev/hack/shell # knative.dev/networking v0.0.0-20201223042504-b9e08949dfbc ## explicit knative.dev/networking/config @@ -1117,6 +1118,7 @@ knative.dev/pkg/test/mako/config knative.dev/pkg/test/monitoring knative.dev/pkg/test/slackutil knative.dev/pkg/test/spoof +knative.dev/pkg/test/upgrade knative.dev/pkg/test/vegeta/pacers knative.dev/pkg/test/zipkin knative.dev/pkg/tracing