From 80a1aff008d91f205e79636463a6c468ce1cb96a Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Fri, 27 Nov 2020 11:35:18 +0100 Subject: [PATCH 01/18] Migrate upgrade tests to using pkg/test/upgrade framework * Convert AssertAutoscaleUpToNumPods and inner functions to helper functions that return error instead of calling t.Fatal to fail the current test. Avoid using *testing.T so that these functions can be reused outside of tests or span multiple tests. * The autoscaler helper functions are used in upgrade tests where "setup" and "verify" phases run within different tests. Pull test.EnsureTearDown from SetupSvc to ensure that a kservice is not destroyed at the end of the first phase ("setup") but remains active until "verify" phase. This is ensured by calling EnsureTearDown later in the "verify" phase. * Adjust Bash scripts to avoid unbound variable errors during upgrade tests exexution (due to using -u flag by the upgrade framework). --- hack/generate-yamls.sh | 3 +- test/e2e-common.sh | 64 ++++-- test/e2e-networking-library.sh | 24 +-- test/e2e-upgrade-tests.sh | 80 +------ test/e2e/autoscale.go | 82 +++---- test/e2e/autoscale_test.go | 41 +++- test/e2e/grpc_test.go | 10 +- test/ha/autoscaler_test.go | 12 +- test/prober.go | 8 +- test/upgrade/autoscaler.go | 96 +++++++++ test/upgrade/autoscaler_test.go | 83 -------- test/upgrade/installation/shell.go | 59 +++++ ...postdowngrade_test.go => postdowngrade.go} | 22 +- ...ice_postupgrade_test.go => postupgrade.go} | 49 +++-- ...rvice_preupgrade_test.go => preupgrade.go} | 42 +++- test/upgrade/probe.go | 62 ++++++ test/upgrade/probe_test.go | 69 ------ test/upgrade/upgrade.go | 18 +- test/upgrade/upgrade_test.go | 79 +++++++ .../knative.dev/hack/shell/LICENSE | 201 ++++++++++++++++++ vendor/knative.dev/hack/shell/executor.go | 189 ++++++++++++++++ vendor/knative.dev/hack/shell/fail-example.sh | 17 ++ vendor/knative.dev/hack/shell/prefixer.go | 68 ++++++ vendor/knative.dev/hack/shell/project.go | 52 +++++ vendor/knative.dev/hack/shell/types.go | 82 +++++++ .../knative.dev/pkg/test/upgrade/functions.go | 162 ++++++++++++++ .../pkg/test/upgrade/private_types.go | 63 ++++++ vendor/knative.dev/pkg/test/upgrade/steps.go | 155 ++++++++++++++ .../pkg/test/upgrade/suite_execution.go | 85 ++++++++ vendor/knative.dev/pkg/test/upgrade/types.go | 123 +++++++++++ vendor/knative.dev/pkg/test/upgrade/vars.go | 32 +++ vendor/modules.txt | 1 + 32 files changed, 1775 insertions(+), 358 deletions(-) create mode 100644 test/upgrade/autoscaler.go delete mode 100644 test/upgrade/autoscaler_test.go create mode 100644 test/upgrade/installation/shell.go rename test/upgrade/{service_postdowngrade_test.go => postdowngrade.go} (80%) rename test/upgrade/{service_postupgrade_test.go => postupgrade.go} (75%) rename test/upgrade/{service_preupgrade_test.go => preupgrade.go} (68%) create mode 100644 test/upgrade/probe.go delete mode 100644 test/upgrade/probe_test.go create mode 100644 test/upgrade/upgrade_test.go create mode 100644 third_party/VENDOR-LICENSE/knative.dev/hack/shell/LICENSE create mode 100644 vendor/knative.dev/hack/shell/executor.go create mode 100644 vendor/knative.dev/hack/shell/fail-example.sh create mode 100644 vendor/knative.dev/hack/shell/prefixer.go create mode 100644 vendor/knative.dev/hack/shell/project.go create mode 100644 vendor/knative.dev/hack/shell/types.go create mode 100644 vendor/knative.dev/pkg/test/upgrade/functions.go create mode 100644 vendor/knative.dev/pkg/test/upgrade/private_types.go create mode 100644 vendor/knative.dev/pkg/test/upgrade/steps.go create mode 100644 vendor/knative.dev/pkg/test/upgrade/suite_execution.go create mode 100644 vendor/knative.dev/pkg/test/upgrade/types.go create mode 100644 vendor/knative.dev/pkg/test/upgrade/vars.go diff --git a/hack/generate-yamls.sh b/hack/generate-yamls.sh index 141adf46264c..3d306c09c84d 100755 --- a/hack/generate-yamls.sh +++ b/hack/generate-yamls.sh @@ -66,6 +66,7 @@ readonly CONSOLIDATED_ARTIFACTS # Flags for all ko commands KO_YAML_FLAGS="-P" +KO_FLAGS="${KO_FLAGS:-}" [[ "${KO_DOCKER_REPO}" != gcr.io/* ]] && KO_YAML_FLAGS="" if [[ "${KO_FLAGS}" != *"--platform"* ]]; then @@ -74,7 +75,7 @@ fi readonly KO_YAML_FLAGS="${KO_YAML_FLAGS} ${KO_FLAGS}" -if [[ -n "${TAG}" ]]; then +if [[ -n "${TAG:-}" ]]; then LABEL_YAML_CMD=(sed -e "s|serving.knative.dev/release: devel|serving.knative.dev/release: \"${TAG}\"|") else LABEL_YAML_CMD=(cat) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 1bc3da3669ec..e474e4e4ff13 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -15,8 +15,9 @@ # limitations under the License. # This script provides helper methods to perform cluster actions. -source $(dirname $0)/../vendor/knative.dev/hack/e2e-tests.sh -source $(dirname $0)/e2e-networking-library.sh +# shellcheck disable=SC1090 +source "$(dirname "${BASH_SOURCE[0]}")/../vendor/knative.dev/hack/e2e-tests.sh" +source "$(dirname "${BASH_SOURCE[0]}")/e2e-networking-library.sh" CERT_MANAGER_VERSION="latest" # Since default is istio, make default ingress as istio @@ -39,12 +40,12 @@ MESH=0 INSTALL_CUSTOM_YAMLS="" UNINSTALL_LIST=() -TMP_DIR=$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX) -readonly TMP_DIR +export TMP_DIR +TMP_DIR="${TMP_DIR:-$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX)}" readonly KNATIVE_DEFAULT_NAMESPACE="knative-serving" # This the namespace used to install Knative Serving. Use generated UUID as namespace. export SYSTEM_NAMESPACE -SYSTEM_NAMESPACE=$(uuidgen | tr 'A-Z' 'a-z') +SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}" # Keep this in sync with test/ha/ha.go @@ -52,6 +53,14 @@ readonly REPLICAS=3 readonly BUCKETS=10 HA_COMPONENTS=() +# Latest serving release. If user does not supply this as a flag, the latest +# tagged release on the current branch will be used. +LATEST_SERVING_RELEASE_VERSION=$(latest_version) + +# Latest net-istio release. +LATEST_NET_ISTIO_RELEASE_VERSION=$( + curl -L --silent "https://api.github.com/repos/knative/net-istio/releases" | grep '"tag_name"' \ + | cut -f2 -d: | sed "s/[^v0-9.]//g" | sort | tail -n1) # Parse our custom flags. function parse_flags() { @@ -141,14 +150,15 @@ function parse_flags() { # All generated YAMLs will be available and pointed by the corresponding # environment variables as set in /hack/generate-yamls.sh. function build_knative_from_source() { - local YAML_LIST="$(mktemp)" + local FULL_OUTPUT YAML_LIST LOG_OUTPUT ENV_OUTPUT + YAML_LIST="$(mktemp)" # Generate manifests, capture environment variables pointing to the YAML files. - local FULL_OUTPUT="$( \ - source $(dirname $0)/../hack/generate-yamls.sh ${REPO_ROOT_DIR} ${YAML_LIST} ; \ + FULL_OUTPUT="$( \ + source "$(dirname "${BASH_SOURCE[0]}")/../hack/generate-yamls.sh" "${REPO_ROOT_DIR}" "${YAML_LIST}" ; \ set | grep _YAML=/)" - local LOG_OUTPUT="$(echo "${FULL_OUTPUT}" | grep -v _YAML=/)" - local ENV_OUTPUT="$(echo "${FULL_OUTPUT}" | grep '^[_0-9A-Z]\+_YAML=/')" + LOG_OUTPUT="$(echo "${FULL_OUTPUT}" | grep -v _YAML=/)" + ENV_OUTPUT="$(echo "${FULL_OUTPUT}" | grep '^[_0-9A-Z]\+_YAML=/')" [[ -z "${LOG_OUTPUT}" || -z "${ENV_OUTPUT}" ]] && fail_test "Error generating manifests" # Only import the environment variables pointing to the YAML files. echo "${LOG_OUTPUT}" @@ -164,7 +174,7 @@ function build_knative_from_source() { function install_knative_serving() { local version=${1:-"HEAD"} if [[ -z "${INSTALL_CUSTOM_YAMLS}" ]]; then - install_knative_serving_standard "$version" "$2" + install_knative_serving_standard "$version" "${2:-}" return fi echo ">> Installing Knative serving from custom YAMLs" @@ -228,15 +238,15 @@ function install_knative_serving_standard() { fi echo ">> Installing Ingress" - if [[ -n "${GLOO_VERSION}" ]]; then + if [[ -n "${GLOO_VERSION:-}" ]]; then install_gloo || return 1 - elif [[ -n "${KOURIER_VERSION}" ]]; then + elif [[ -n "${KOURIER_VERSION:-}" ]]; then install_kourier || return 1 - elif [[ -n "${AMBASSADOR_VERSION}" ]]; then + elif [[ -n "${AMBASSADOR_VERSION:-}" ]]; then install_ambassador || return 1 - elif [[ -n "${CONTOUR_VERSION}" ]]; then + elif [[ -n "${CONTOUR_VERSION:-}" ]]; then install_contour || return 1 - elif [[ -n "${KONG_VERSION}" ]]; then + elif [[ -n "${KONG_VERSION:-}" ]]; then install_kong || return 1 else if [[ "$1" == "HEAD" ]]; then @@ -511,3 +521,25 @@ function disable_chaosduck() { function enable_chaosduck() { kubectl -n "${SYSTEM_NAMESPACE}" scale deployment "chaosduck" --replicas=1 || fail_test } + +function install_latest_release() { + header "Installing Knative latest public release" + + install_knative_serving latest-release \ + || fail_test "Knative latest release installation failed" + test_logging_config_setup + wait_until_pods_running ${SYSTEM_NAMESPACE} + wait_until_batch_job_complete ${SYSTEM_NAMESPACE} +} + +function install_head() { + header "Installing Knative head release" + install_knative_serving || fail_test "Knative head release installation failed" + test_logging_config_setup + wait_until_pods_running ${SYSTEM_NAMESPACE} + wait_until_batch_job_complete ${SYSTEM_NAMESPACE} +} + +function knative_setup() { + install_latest_release +} diff --git a/test/e2e-networking-library.sh b/test/e2e-networking-library.sh index 635cab9fab69..972f0148fa8e 100644 --- a/test/e2e-networking-library.sh +++ b/test/e2e-networking-library.sh @@ -15,17 +15,17 @@ # limitations under the License. function install_istio() { - if [[ -z "${ISTIO_VERSION}" ]]; then + if [[ -z "${ISTIO_VERSION:-}" ]]; then readonly ISTIO_VERSION="stable" fi - if [[ -z "${NET_ISTIO_COMMIT}" ]]; then + if [[ -z "${NET_ISTIO_COMMIT:-}" ]]; then NET_ISTIO_COMMIT=$(head -n 1 ${1} | grep "# Generated when HEAD was" | sed 's/^.* //') echo "Got NET_ISTIO_COMMIT from ${1}: ${NET_ISTIO_COMMIT}" fi # TODO: remove this when all the net-istio.yaml in use contain a commit ID - if [[ -z "${NET_ISTIO_COMMIT}" ]]; then + if [[ -z "${NET_ISTIO_COMMIT:-}" ]]; then NET_ISTIO_COMMIT="8102cd3d32f05be1c58260a9717d532a4a6d2f60" echo "Hard coded NET_ISTIO_COMMIT: ${NET_ISTIO_COMMIT}" fi @@ -41,7 +41,7 @@ function install_istio() { ) ISTIO_PROFILE="istio" - if [[ -n "$KIND" ]]; then + if [[ -n "${KIND:-}" ]]; then ISTIO_PROFILE+="-kind" else ISTIO_PROFILE+="-ci" @@ -52,7 +52,7 @@ function install_istio() { ISTIO_PROFILE+="-mesh" ISTIO_PROFILE+=".yaml" - if [[ -n "$CLUSTER_DOMAIN" ]]; then + if [[ -n "${CLUSTER_DOMAIN:-}" ]]; then sed -ie "s/cluster\.local/${CLUSTER_DOMAIN}/g" ${NET_ISTIO_DIR}/third_party/istio-${ISTIO_VERSION}/${ISTIO_PROFILE} fi @@ -61,7 +61,7 @@ function install_istio() { echo "Istio profile: ${ISTIO_PROFILE}" ${NET_ISTIO_DIR}/third_party/istio-${ISTIO_VERSION}/install-istio.sh ${ISTIO_PROFILE} - if [[ -n "$1" ]]; then + if [[ -n "${1:-}" ]]; then echo ">> Installing net-istio" echo "net-istio original YAML: ${1}" # Create temp copy in which we replace knative-serving by the test's system namespace. @@ -161,11 +161,11 @@ function install_contour() { } function wait_until_ingress_running() { - if [[ -n "${ISTIO_VERSION}" ]]; then + if [[ -n "${ISTIO_VERSION:-}" ]]; then wait_until_pods_running istio-system || return 1 wait_until_service_has_external_http_address istio-system istio-ingressgateway || return 1 fi - if [[ -n "${GLOO_VERSION}" ]]; then + if [[ -n "${GLOO_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Gloo # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=knative-external-proxy @@ -173,7 +173,7 @@ function wait_until_ingress_running() { wait_until_pods_running gloo-system || return 1 wait_until_service_has_external_ip gloo-system knative-external-proxy fi - if [[ -n "${KOURIER_VERSION}" ]]; then + if [[ -n "${KOURIER_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Kourier # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=kourier @@ -181,7 +181,7 @@ function wait_until_ingress_running() { wait_until_pods_running kourier-system || return 1 wait_until_service_has_external_http_address kourier-system kourier fi - if [[ -n "${AMBASSADOR_VERSION}" ]]; then + if [[ -n "${AMBASSADOR_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Ambassador # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=ambassador @@ -189,7 +189,7 @@ function wait_until_ingress_running() { wait_until_pods_running ambassador || return 1 wait_until_service_has_external_http_address ambassador ambassador fi - if [[ -n "${CONTOUR_VERSION}" ]]; then + if [[ -n "${CONTOUR_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Contour # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=envoy @@ -198,7 +198,7 @@ function wait_until_ingress_running() { wait_until_pods_running contour-internal || return 1 wait_until_service_has_external_ip "${GATEWAY_NAMESPACE_OVERRIDE}" "${GATEWAY_OVERRIDE}" fi - if [[ -n "${KONG_VERSION}" ]]; then + if [[ -n "${KONG_VERSION:-}" ]]; then # we must set these override values to allow the test spoofing client to work with Kong # see https://github.com/knative/pkg/blob/release-0.7/test/ingress/ingress.go#L37 export GATEWAY_OVERRIDE=kong-proxy diff --git a/test/e2e-upgrade-tests.sh b/test/e2e-upgrade-tests.sh index 5c37169f5868..35fc8e5ea157 100755 --- a/test/e2e-upgrade-tests.sh +++ b/test/e2e-upgrade-tests.sh @@ -28,39 +28,8 @@ # You can specify the version to run against with the --version argument # (e.g. --version v0.7.0). If this argument is not specified, the script will # run against the latest tagged version on the current branch. - -source $(dirname $0)/e2e-common.sh - -# Latest serving release. If user does not supply this as a flag, the latest -# tagged release on the current branch will be used. -LATEST_SERVING_RELEASE_VERSION=$(latest_version) - -# Latest net-istio release. -LATEST_NET_ISTIO_RELEASE_VERSION=$( - curl -L --silent "https://api.github.com/repos/knative/net-istio/releases" | grep '"tag_name"' \ - | cut -f2 -d: | sed "s/[^v0-9.]//g" | sort | tail -n1) - -function install_latest_release() { - header "Installing Knative latest public release" - - install_knative_serving latest-release \ - || fail_test "Knative latest release installation failed" - test_logging_config_setup - wait_until_pods_running ${SYSTEM_NAMESPACE} - wait_until_batch_job_complete ${SYSTEM_NAMESPACE} -} - -function install_head() { - header "Installing Knative head release" - install_knative_serving || fail_test "Knative head release installation failed" - test_logging_config_setup - wait_until_pods_running ${SYSTEM_NAMESPACE} - wait_until_batch_job_complete ${SYSTEM_NAMESPACE} -} - -function knative_setup() { - install_latest_release -} +# shellcheck disable=SC1090 +source "$(dirname "${BASH_SOURCE[0]}")/e2e-common.sh" # Script entry point. @@ -75,51 +44,14 @@ initialize "$@" --skip-istio-addon --min-nodes=4 --max-nodes=4 disable_chaosduck # TODO(#2656): Reduce the timeout after we get this test to consistently passing. -TIMEOUT=10m -# Probe tests starts before postupgrade tests and ends after postdowngrade tests. -# The timeout should be at least 10m + 10m + installation time -PROBE_TIMEOUT=20m +TIMEOUT=30m -header "Running preupgrade tests" +header "Running upgrade tests" -go_test_e2e -tags=preupgrade -timeout=${TIMEOUT} ./test/upgrade \ +go_test_e2e -tags=upgrade -timeout=${TIMEOUT} \ + ./test/upgrade \ --resolvabledomain=$(use_resolvable_domain) || fail_test -header "Starting prober test" - -# Remove the following files in case we failed to clean them up in an earlier test. -rm -f /tmp/prober-signal -rm -f /tmp/autoscaling-signal -rm -f /tmp/autoscaling-tbc-signal - -go_test_e2e -tags=probe -timeout=${PROBE_TIMEOUT} ./test/upgrade \ - --resolvabledomain=$(use_resolvable_domain) & -PROBER_PID=$! -echo "Prober PID is ${PROBER_PID}" - -install_head - -header "Running postupgrade tests" -go_test_e2e -tags=postupgrade -timeout=${TIMEOUT} ./test/upgrade \ - --resolvabledomain=$(use_resolvable_domain) || fail_test - -install_latest_release - -header "Running postdowngrade tests" -go_test_e2e -tags=postdowngrade -timeout=${TIMEOUT} ./test/upgrade \ - --resolvabledomain=$(use_resolvable_domain) || fail_test - -# The probe tests are blocking on the following files to know when it should exit. -# -# This is kind of gross. First attempt was to just send a signal to the go test, -# but "go test" intercepts the signal and always exits with a non-zero code. -echo "done" > /tmp/prober-signal -echo "done" > /tmp/autoscaling-signal -echo "done" > /tmp/autoscaling-tbc-signal - -header "Waiting for prober test" -wait ${PROBER_PID} || fail_test "Prober failed" - # Remove the kail log file if the test flow passes. # This is for preventing too many large log files to be uploaded to GCS in CI. rm "${KAIL_LOG_FILE}" diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index 7f1b1e5762f2..a8efb6bdf3c3 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -27,6 +27,8 @@ import ( "testing" "time" + "knative.dev/pkg/test/logging" + vegeta "github.com/tsenart/vegeta/v12/lib" "golang.org/x/sync/errgroup" corev1 "k8s.io/api/core/v1" @@ -61,7 +63,7 @@ const ( type TestContext struct { t *testing.T clients *test.Clients - names test.ResourceNames + names *test.ResourceNames resources *v1test.ResourceObjects targetUtilization float64 targetValue int @@ -84,12 +86,12 @@ func (ctx *TestContext) SetResources(resources *v1test.ResourceObjects) { } // Names returns the resource names of the TestContext. -func (ctx *TestContext) Names() test.ResourceNames { +func (ctx *TestContext) Names() *test.ResourceNames { return ctx.names } // SetNames set the resource names of the TestContext to the given values. -func (ctx *TestContext) SetNames(names test.ResourceNames) { +func (ctx *TestContext) SetNames(names *test.ResourceNames) { ctx.names = names } @@ -120,6 +122,7 @@ func getVegetaTarget(kubeClientset kubernetes.Interface, domain, endpointOverrid func generateTraffic( ctx *TestContext, + logf logging.FormatLogger, attacker *vegeta.Attacker, pacer vegeta.Pacer, stopChan chan struct{}) error { @@ -141,7 +144,7 @@ func generateTraffic( for { select { case <-stopChan: - ctx.t.Log("Stopping generateTraffic") + logf("Stopping generateTraffic") successRate := float64(1) if totalRequests > 0 { successRate = float64(successfulRequests) / float64(totalRequests) @@ -153,14 +156,14 @@ func generateTraffic( return nil case res, ok := <-results: if !ok { - ctx.t.Log("Time is up; done") + logf("Time is up; done") return nil } totalRequests++ if res.Code != http.StatusOK { - ctx.t.Logf("Status = %d, want: 200", res.Code) - ctx.t.Logf("URL: %s Duration: %v Error: %s Body:\n%s", res.URL, res.Latency, res.Error, string(res.Body)) + logf("Status = %d, want: 200", res.Code) + logf("URL: %s Duration: %v Error: %s Body:\n%s", res.URL, res.Latency, res.Error, string(res.Body)) continue } successfulRequests++ @@ -168,23 +171,23 @@ func generateTraffic( } } -func generateTrafficAtFixedConcurrency(ctx *TestContext, concurrency int, stopChan chan struct{}) error { +func generateTrafficAtFixedConcurrency(ctx *TestContext, logf logging.FormatLogger, concurrency int, stopChan chan struct{}) error { pacer := vegeta.ConstantPacer{} // Sends requests as quickly as possible, capped by MaxWorkers below. attacker := vegeta.NewAttacker( vegeta.Timeout(0), // No timeout is enforced at all. vegeta.Workers(uint64(concurrency)), vegeta.MaxWorkers(uint64(concurrency))) - ctx.t.Logf("Maintaining %d concurrent requests.", concurrency) - return generateTraffic(ctx, attacker, pacer, stopChan) + logf("Maintaining %d concurrent requests.", concurrency) + return generateTraffic(ctx, logf, attacker, pacer, stopChan) } -func generateTrafficAtFixedRPS(ctx *TestContext, rps int, stopChan chan struct{}) error { +func generateTrafficAtFixedRPS(ctx *TestContext, logf logging.FormatLogger, rps int, stopChan chan struct{}) error { pacer := vegeta.ConstantPacer{Freq: rps, Per: time.Second} attacker := vegeta.NewAttacker(vegeta.Timeout(0)) // No timeout is enforced at all. - ctx.t.Logf("Maintaining %v RPS.", rps) - return generateTraffic(ctx, attacker, pacer, stopChan) + logf("Maintaining %v RPS.", rps) + return generateTraffic(ctx, logf, attacker, pacer, stopChan) } func toPercentageString(f float64) string { @@ -201,12 +204,11 @@ func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization clients := Setup(t) t.Log("Creating a new Route and Configuration") - names := test.ResourceNames{ + names := &test.ResourceNames{ Service: test.ObjectNameForTest(t), Image: autoscaleTestImageName, } - test.EnsureTearDown(t, clients, &names) - resources, err := v1test.CreateServiceReady(t, clients, &names, + resources, err := v1test.CreateServiceReady(t, clients, names, append([]rtesting.ServiceOption{ rtesting.WithConfigAnnotations(map[string]string{ autoscaling.ClassAnnotationKey: class, @@ -258,14 +260,14 @@ func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization } } -func assertScaleDown(ctx *TestContext) { +func assertScaleDown(ctx *TestContext, logf logging.FormatLogger) error { deploymentName := resourcenames.Deployment(ctx.resources.Revision) if err := WaitForScaleToZero(ctx.t, deploymentName, ctx.clients); err != nil { - ctx.t.Fatalf("Unable to observe the Deployment named %s scaling down: %v", deploymentName, err) + return fmt.Errorf("unable to observe the Deployment named %s scaling down: %w", deploymentName, err) } // Account for the case where scaling up uses all available pods. - ctx.t.Log("Wait for all pods to terminate.") + logf("Wait for all pods to terminate.") if err := pkgTest.WaitForPodListState( context.Background(), @@ -281,27 +283,29 @@ func assertScaleDown(ctx *TestContext) { return true, nil }, "WaitForAvailablePods", test.ServingNamespace); err != nil { - ctx.t.Fatalf("Waiting for Pod.List to have no non-Evicted pods of %q: %v", deploymentName, err) + return fmt.Errorf("waiting for Pod.List to have no non-Evicted pods of %q: %w", deploymentName, err) } - ctx.t.Log("The Revision should remain ready after scaling to zero.") + logf("The Revision should remain ready after scaling to zero.") if err := v1test.CheckRevisionState(ctx.clients.ServingClient, ctx.names.Revision, v1test.IsRevisionReady); err != nil { - ctx.t.Fatalf("The Revision %s did not stay Ready after scaling down to zero: %v", ctx.names.Revision, err) + return fmt.Errorf("the Revision %s did not stay Ready after scaling down to zero: %w", ctx.names.Revision, err) } - ctx.t.Log("Scaled down.") + logf("Scaled down.") + + return nil } -func numberOfReadyPods(ctx *TestContext) (float64, error) { +func numberOfReadyPods(ctx *TestContext, logf logging.FormatLogger) (float64, error) { // SKS name matches that of revision. n := ctx.resources.Revision.Name sks, err := ctx.clients.NetworkingClient.ServerlessServices.Get(context.Background(), n, metav1.GetOptions{}) if err != nil { - ctx.t.Logf("Error getting SKS %q: %v", n, err) + logf("Error getting SKS %q: %v", n, err) return 0, fmt.Errorf("error retrieving sks %q: %w", n, err) } if sks.Status.PrivateServiceName == "" { - ctx.t.Logf("SKS %s has not yet reconciled", n) + logf("SKS %s has not yet reconciled", n) // Not an error, but no pods either. return 0, nil } @@ -313,7 +317,7 @@ func numberOfReadyPods(ctx *TestContext) (float64, error) { return float64(resources.ReadyAddressCount(eps)), nil } -func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done <-chan time.Time, quick bool) error { +func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minPods, maxPods float64, done <-chan time.Time, quick bool) error { // Short-circuit traffic generation once we exit from the check logic. ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() @@ -323,12 +327,12 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done case <-ticker.C: // Each 2 second, check that the number of pods is at least `minPods`. `minPods` is increasing // to verify that the number of pods doesn't go down while we are scaling up. - got, err := numberOfReadyPods(ctx) + got, err := numberOfReadyPods(ctx, logf) if err != nil { return err } mes := fmt.Sprintf("revision %q #replicas: %v, want at least: %v", ctx.resources.Revision.Name, got, minPods) - ctx.t.Log(mes) + logf(mes) // verify that the number of pods doesn't go down while we are scaling up. if got < minPods { return errors.New("interim scale didn't fulfill constraints: " + mes) @@ -336,7 +340,7 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done // A quick test succeeds when the number of pods scales up to `targetPods` // (and, as an extra check, no more than `maxPods`). if quick && got >= targetPods && got <= maxPods { - ctx.t.Logf("Quick Mode: got %v >= %v", got, targetPods) + logf("Quick Mode: got %v >= %v", got, targetPods) return nil } if minPods < targetPods-1 { @@ -347,13 +351,13 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done case <-done: // The test duration is over. Do a last check to verify that the number of pods is at `targetPods` // (with a little room for de-flakiness). - got, err := numberOfReadyPods(ctx) + got, err := numberOfReadyPods(ctx, logf) if err != nil { return fmt.Errorf("failed to fetch number of ready pods: %w", err) } mes := fmt.Sprintf("got %v replicas, expected between [%v, %v] replicas for revision %s", got, targetPods-1, maxPods, ctx.resources.Revision.Name) - ctx.t.Log(mes) + logf(mes) if got < targetPods-1 || got > maxPods { return errors.New("final scale didn't fulfill constraints: " + mes) } @@ -369,9 +373,7 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done // sustains there until the `done` channel sends a signal. // The given `duration` is how long the traffic will be generated. You must make sure that the signal // from the given `done` channel will be sent within the `duration`. -func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) { - ctx.t.Helper() - +func AssertAutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, targetPods float64, done <-chan time.Time, quick bool) error { // Relax the bounds to reduce the flakiness caused by sampling in the autoscaling algorithm. // Also adjust the values by the target utilization values. minPods := math.Floor(curPods/ctx.targetUtilization) - 1 @@ -382,18 +384,20 @@ func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, d grp.Go(func() error { switch ctx.metric { case autoscaling.RPS: - return generateTrafficAtFixedRPS(ctx, int(targetPods*float64(ctx.targetValue)), stopChan) + return generateTrafficAtFixedRPS(ctx, logf, int(targetPods*float64(ctx.targetValue)), stopChan) default: - return generateTrafficAtFixedConcurrency(ctx, int(targetPods*float64(ctx.targetValue)), stopChan) + return generateTrafficAtFixedConcurrency(ctx, logf, int(targetPods*float64(ctx.targetValue)), stopChan) } }) grp.Go(func() error { defer close(stopChan) - return checkPodScale(ctx, targetPods, minPods, maxPods, done, quick) + return checkPodScale(ctx, logf, targetPods, minPods, maxPods, done, quick) }) if err := grp.Wait(); err != nil { - ctx.t.Fatal("Error: ", err) + return err } + + return nil } diff --git a/test/e2e/autoscale_test.go b/test/e2e/autoscale_test.go index 387f7eb223db..d967b9bfd369 100644 --- a/test/e2e/autoscale_test.go +++ b/test/e2e/autoscale_test.go @@ -41,10 +41,17 @@ func TestAutoscaleUpDownUp(t *testing.T) { t.Parallel() ctx := SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) - AssertAutoscaleUpToNumPods(ctx, 1, 2, time.After(60*time.Second), true /* quick */) - assertScaleDown(ctx) - AssertAutoscaleUpToNumPods(ctx, 0, 2, time.After(60*time.Second), true /* quick */) + if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 2, time.After(60*time.Second), true /* quick */); err != nil { + t.Fatal(err) + } + if err := assertScaleDown(ctx, t.Logf); err != nil { + t.Fatal(err) + } + if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 0, 2, time.After(60*time.Second), true /* quick */); err != nil { + t.Fatal(err) + } } func TestAutoscaleUpCountPods(t *testing.T) { @@ -66,6 +73,7 @@ func runAutoscaleUpCountPods(t *testing.T, class, metric string) { } ctx := SetupSvc(t, class, metric, target, targetUtilization) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) ctx.t.Log("The autoscaler spins up additional replicas when traffic increases.") // Note: without the warm-up / gradual increase of load the test is @@ -75,11 +83,17 @@ func runAutoscaleUpCountPods(t *testing.T, class, metric string) { // boskos cluster to propagate the state. See #10218. // Assert the number of expected replicas is between n-1 and n+1, where n is the # of desired replicas for 60s. // Assert the number of expected replicas is n and n+1 at the end of 90s, where n is the # of desired replicas. - AssertAutoscaleUpToNumPods(ctx, 1, 2, time.After(90*time.Second), true /* quick */) + if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 2, time.After(90*time.Second), true /* quick */); err != nil { + t.Fatal(err) + } // Increase workload scale to 3 replicas, assert between [n-1, n+1] during scale up, assert between [n, n+1] after scaleup. - AssertAutoscaleUpToNumPods(ctx, 2, 3, time.After(90*time.Second), true /* quick */) + if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 2, 3, time.After(90*time.Second), true /* quick */); err != nil { + t.Fatal(err) + } // Increase workload scale to 4 replicas, assert between [n-1, n+1] during scale up, assert between [n, n+1] after scaleup. - AssertAutoscaleUpToNumPods(ctx, 3, 4, time.After(90*time.Second), true /* quick */) + if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 3, 4, time.After(90*time.Second), true /* quick */); err != nil { + t.Fatal(err) + } } func TestAutoscaleSustaining(t *testing.T) { @@ -89,7 +103,11 @@ func TestAutoscaleSustaining(t *testing.T) { t.Parallel() ctx := SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization) - AssertAutoscaleUpToNumPods(ctx, 1, 10, time.After(2*time.Minute), false /* quick */) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) + + if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 10, time.After(2*time.Minute), false /* quick */); err != nil { + t.Fatal(err) + } } func TestTargetBurstCapacity(t *testing.T) { @@ -105,6 +123,7 @@ func TestTargetBurstCapacity(t *testing.T) { autoscaling.TargetBurstCapacityKey: "7", autoscaling.PanicThresholdPercentageAnnotationKey: "200", // makes panicking rare })) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) cfg, err := autoscalerCM(ctx.clients) if err != nil { @@ -118,7 +137,7 @@ func TestTargetBurstCapacity(t *testing.T) { defer close(stopCh) grp.Go(func() error { - return generateTrafficAtFixedConcurrency(ctx, 7, stopCh) + return generateTrafficAtFixedConcurrency(ctx, t.Logf, 7, stopCh) }) // Wait for the activator endpoints to equalize. @@ -128,13 +147,13 @@ func TestTargetBurstCapacity(t *testing.T) { // Start second load generator. grp.Go(func() error { - return generateTrafficAtFixedConcurrency(ctx, 5, stopCh) + return generateTrafficAtFixedConcurrency(ctx, t.Logf, 5, stopCh) }) // Wait for two stable pods. obsScale := 0.0 if err := wait.Poll(250*time.Millisecond, 2*cfg.StableWindow, func() (bool, error) { - obsScale, err = numberOfReadyPods(ctx) + obsScale, err = numberOfReadyPods(ctx, t.Logf) if err != nil { return false, err } @@ -167,6 +186,7 @@ func TestTargetBurstCapacityMinusOne(t *testing.T) { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "-1", })) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) _, err := autoscalerCM(ctx.clients) if err != nil { @@ -193,6 +213,7 @@ func TestFastScaleToZero(t *testing.T) { autoscaling.TargetBurstCapacityKey: "-1", autoscaling.WindowAnnotationKey: autoscaling.WindowMin.String(), })) + test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) cfg, err := autoscalerCM(ctx.clients) if err != nil { diff --git a/test/e2e/grpc_test.go b/test/e2e/grpc_test.go index cc15ea03f424..29b3917d7768 100644 --- a/test/e2e/grpc_test.go +++ b/test/e2e/grpc_test.go @@ -111,7 +111,7 @@ func autoscaleTest(ctx *TestContext, host, domain string) { ctx.targetUtilization = targetUtilization assertGRPCAutoscaleUpToNumPods(ctx, 1, 2, 60*time.Second, host, domain) - assertScaleDown(ctx) + assertScaleDown(ctx, ctx.t.Logf) assertGRPCAutoscaleUpToNumPods(ctx, 0, 2, 60*time.Second, host, domain) } @@ -256,7 +256,7 @@ func assertGRPCAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float6 grp.Go(func() error { defer close(stopChan) - return checkPodScale(ctx, targetPods, minPods, maxPods, time.After(duration), true /* quick */) + return checkPodScale(ctx, ctx.t.Logf, targetPods, minPods, maxPods, time.After(duration), true /* quick */) }) if err := grp.Wait(); err != nil { @@ -324,15 +324,15 @@ func testGRPC(t *testing.T, f grpcTest, fopts ...rtesting.ServiceOption) { t.Log("Creating service for grpc-ping") - names := test.ResourceNames{ + names := &test.ResourceNames{ Service: test.ObjectNameForTest(t), Image: "grpc-ping", } fopts = append(fopts, rtesting.WithNamedPort("h2c")) - test.EnsureTearDown(t, clients, &names) - resources, err := v1test.CreateServiceReady(t, clients, &names, fopts...) + test.EnsureTearDown(t, clients, names) + resources, err := v1test.CreateServiceReady(t, clients, names, fopts...) if err != nil { t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err) } diff --git a/test/ha/autoscaler_test.go b/test/ha/autoscaler_test.go index 43f52e0f7744..0d8e108daf39 100644 --- a/test/ha/autoscaler_test.go +++ b/test/ha/autoscaler_test.go @@ -55,7 +55,7 @@ func TestAutoscalerHA(t *testing.T) { resources := ctx.Resources() clients := ctx.Clients() - test.EnsureTearDown(t, clients, &names) + test.EnsureTearDown(t, clients, names) t.Log("Expected replicas = ", test.ServingFlags.Replicas) if err := pkgTest.WaitForDeploymentScale(context.Background(), clients.KubeClient, autoscalerDeploymentName, system.Namespace(), test.ServingFlags.Replicas); err != nil { @@ -90,7 +90,9 @@ func TestAutoscalerHA(t *testing.T) { } t.Log("Verifying the old revision can still be scaled from 0 to some number after leader change") - e2e.AssertAutoscaleUpToNumPods(ctx, 0, 3, time.After(60*time.Second), true /* quick */) + if err := e2e.AssertAutoscaleUpToNumPods(ctx, t.Logf, 0, 3, time.After(60*time.Second), true /* quick */); err != nil { + t.Fatal(err) + } t.Log("Updating the Service after selecting new leader controller in order to generate a new revision") names.Image = test.PizzaPlanet2 @@ -100,7 +102,7 @@ func TestAutoscalerHA(t *testing.T) { } t.Log("Service should be able to generate a new revision after changing the leader controller") - names.Revision, err = v1test.WaitForServiceLatestRevision(clients, names) + names.Revision, err = v1test.WaitForServiceLatestRevision(clients, *names) if err != nil { t.Fatal("New image not reflected in Service:", err) } @@ -109,5 +111,7 @@ func TestAutoscalerHA(t *testing.T) { t.Log("Verifying the new revision can be scaled up") ctx.SetNames(names) ctx.SetResources(resources) - e2e.AssertAutoscaleUpToNumPods(ctx, 1, 3, time.After(60*time.Second), true /* quick */) + if err := e2e.AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 3, time.After(60*time.Second), true /* quick */); err != nil { + t.Fatal(err) + } } diff --git a/test/prober.go b/test/prober.go index 893453e56e00..b2d1c31e5e6c 100644 --- a/test/prober.go +++ b/test/prober.go @@ -233,12 +233,18 @@ func RunRouteProber(logf logging.FormatLogger, clients *Clients, url *url.URL, o // against the default SLO, which requires perfect responses. // This takes `testing.T` so that it may be used in `defer`. func AssertProberDefault(t testing.TB, p Prober) { + AssertProberSLO(t, p, 1.0) +} + +// AssertProberSLO is a helper for stopping the Prober and checking its SLI +// against the given SLO. +func AssertProberSLO(t testing.TB, p Prober, slo float64) { t.Helper() if err := p.Stop(); err != nil { t.Error("Stop()", "error", err.Error()) } // Default to 100% correct (typically used in conjunction with the low probe count above) - if err := CheckSLO(1.0, t.Name(), p); err != nil { + if err := CheckSLO(slo, t.Name(), p); err != nil { t.Error("CheckSLO()", "error", err.Error()) } } diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go new file mode 100644 index 000000000000..58a8b4c7b511 --- /dev/null +++ b/test/upgrade/autoscaler.go @@ -0,0 +1,96 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "time" + + "go.uber.org/zap" + + "golang.org/x/sync/errgroup" + "knative.dev/serving/test" + + pkgupgrade "knative.dev/pkg/test/upgrade" + + "knative.dev/serving/pkg/apis/autoscaling" + rtesting "knative.dev/serving/pkg/testing/v1" + "knative.dev/serving/test/e2e" +) + +const ( + containerConcurrency = 6 + targetUtilization = 0.7 +) + +// AutoscaleSustainingTest checks that when traffic increases a knative app +// scales up and sustains the scale as long as the traffic sustains, despite whether +// it is switching modes between normal and panic. +func AutoscaleSustainingTest(logger *zap.Logger) pkgupgrade.BackgroundOperation { + var ctx *e2e.TestContext + var grp errgroup.Group + stopCh := make(chan time.Time) + return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingTest", + func(c pkgupgrade.Context) { + // Setup + ctx = e2e.SetupSvc(c.T, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, + rtesting.WithConfigAnnotations(map[string]string{ + autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. + })) + grp.Go(func() error { + return e2e.AssertAutoscaleUpToNumPods(ctx, logger.Sugar().Infof, 1, 10, stopCh, false /* quick */) + }) + }, + func(c pkgupgrade.Context) { + test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) + // Verification is done inside e2e.AssertAutoscaleUpToNumPods. + // We're just giving it a signal. + close(stopCh) + if err := grp.Wait(); err != nil { + c.T.Error("Error: ", err) + } + }, + ) +} + +// AutoscaleSustainingWithTBCTest checks that when traffic increases and the activator is +// in the path a knative app scales up and sustains the scale. +func AutoscaleSustainingWithTBCTest(logger *zap.Logger) pkgupgrade.BackgroundOperation { + var ctx *e2e.TestContext + var grp errgroup.Group + stopCh := make(chan time.Time) + return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingWithTBCTest", + func(c pkgupgrade.Context) { + // Setup + ctx = e2e.SetupSvc(c.T, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, + rtesting.WithConfigAnnotations(map[string]string{ + autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. + })) + grp.Go(func() error { + return e2e.AssertAutoscaleUpToNumPods(ctx, logger.Sugar().Infof, 1, 10, stopCh, false /* quick */) + }) + }, + func(c pkgupgrade.Context) { + test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) + // Verification is done inside e2e.AssertAutoscaleUpToNumPods. + // We're just giving it a signal. + close(stopCh) + if err := grp.Wait(); err != nil { + c.T.Error("Error: ", err) + } + }, + ) +} diff --git a/test/upgrade/autoscaler_test.go b/test/upgrade/autoscaler_test.go deleted file mode 100644 index 533d3c8a1425..000000000000 --- a/test/upgrade/autoscaler_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// +build probe - -/* -Copyright 2020 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "io/ioutil" - "testing" - "time" - - "knative.dev/serving/pkg/apis/autoscaling" - rtesting "knative.dev/serving/pkg/testing/v1" - "knative.dev/serving/test/e2e" -) - -const ( - containerConcurrency = 6 - targetUtilization = 0.7 - - autoscalingPipe = "/tmp/autoscaling-signal" - autoscalingTBCPipe = "/tmp/autoscaling-tbc-signal" -) - -// This test similar to TestAutoscaleSustaining in test/e2e/autoscale_test.go. It asserts -// the pods number is sustained during the whole cluster upgrade/downgrade process. -func TestAutoscaleSustaining(t *testing.T) { - t.Parallel() - // Create a named pipe and wait for the upgrade script to write to it - // to signal that we should stop testing. - createPipe(t, autoscalingPipe) - - ctx := e2e.SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, - rtesting.WithConfigAnnotations(map[string]string{ - autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. - })) - - stopCh := make(chan time.Time) - go func() { - // e2e-upgrade-test.sh will close this pipe to signal the upgrade is - // over, at which point we will finish the test. - ioutil.ReadFile(autoscalingPipe) - close(stopCh) - }() - - e2e.AssertAutoscaleUpToNumPods(ctx, 1, 10, stopCh, false /* quick */) -} - -func TestAutoscaleSustainingWithTBC(t *testing.T) { - t.Parallel() - // Create a named pipe and wait for the upgrade script to write to it - // to signal that we should stop testing. - createPipe(t, autoscalingTBCPipe) - - ctx := e2e.SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization, - rtesting.WithConfigAnnotations(map[string]string{ - autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. - })) - - stopCh := make(chan time.Time) - go func() { - // e2e-upgrade-test.sh will close this pipe to signal the upgrade is - // over, at which point we will finish the test. - ioutil.ReadFile(autoscalingTBCPipe) - close(stopCh) - }() - - e2e.AssertAutoscaleUpToNumPods(ctx, 1, 10, stopCh, false /* quick */) -} diff --git a/test/upgrade/installation/shell.go b/test/upgrade/installation/shell.go new file mode 100644 index 000000000000..c3f67f502e23 --- /dev/null +++ b/test/upgrade/installation/shell.go @@ -0,0 +1,59 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installation + +import ( + "knative.dev/hack/shell" + pkgupgrade "knative.dev/pkg/test/upgrade" +) + +// Head installs Knative Serving from the HEAD of the master branch. +func Head() pkgupgrade.Operation { + return install("ServingHead", "install_head") +} + +// LatestRelease installs Knative Serving from the latest stable release. +func LatestRelease() pkgupgrade.Operation { + return install("ServingLatestRelease", "install_latest_release") +} + +func install(installName, shellFunc string) pkgupgrade.Operation { + return pkgupgrade.NewOperation(installName, func(c pkgupgrade.Context) { + c.Log.Info("Running shell function: ", shellFunc) + if err := callShellFunction(shellFunc); err != nil { + c.T.Error(err) + } + }) +} + +func callShellFunction(funcName string) error { + loc, err := shell.NewProjectLocation("../../..") + if err != nil { + return err + } + exec := shell.NewExecutor(shell.ExecutorConfig{ + ProjectLocation: loc, + }) + fn := shell.Function{ + Script: shell.Script{ + Label: funcName, + ScriptPath: "test/e2e-common.sh", + }, + FunctionName: funcName, + } + return exec.RunFunction(fn) +} diff --git a/test/upgrade/service_postdowngrade_test.go b/test/upgrade/postdowngrade.go similarity index 80% rename from test/upgrade/service_postdowngrade_test.go rename to test/upgrade/postdowngrade.go index 26d1df3c5a31..d663b33bd6de 100644 --- a/test/upgrade/service_postdowngrade_test.go +++ b/test/upgrade/postdowngrade.go @@ -1,5 +1,3 @@ -// +build postdowngrade - /* Copyright 2018 The Knative Authors @@ -23,9 +21,9 @@ import ( "testing" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - _ "knative.dev/pkg/system/testing" ptest "knative.dev/pkg/test" + pkgupgrade "knative.dev/pkg/test/upgrade" serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names" rtesting "knative.dev/serving/pkg/testing/v1" "knative.dev/serving/test" @@ -33,7 +31,15 @@ import ( v1test "knative.dev/serving/test/v1" ) -func TestServicePostDowngrade(t *testing.T) { +// ServicePostDowngradeTest verifies an existing service after downgrade. +func ServicePostDowngradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePostDowngradeTest", func(c pkgupgrade.Context) { + servicePostDowngrade(c.T) + }) +} + +func servicePostDowngrade(t *testing.T) { + t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -74,7 +80,9 @@ func TestServicePostDowngrade(t *testing.T) { assertServiceResourcesUpdated(t, clients, names, url.URL(), test.PizzaPlanetText1) } -func TestCreateNewServicePostDowngrade(t *testing.T) { - t.Parallel() - createNewService(postDowngradeServiceName, t) +// CreateNewServicePostDowngradeTest verifies creating a new service after downgrade. +func CreateNewServicePostDowngradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("CreateNewServicePostDowngradeTest", func(c pkgupgrade.Context) { + createNewService(postDowngradeServiceName, c.T) + }) } diff --git a/test/upgrade/service_postupgrade_test.go b/test/upgrade/postupgrade.go similarity index 75% rename from test/upgrade/service_postupgrade_test.go rename to test/upgrade/postupgrade.go index d26f8bfeb19b..e32bd61b173b 100644 --- a/test/upgrade/service_postupgrade_test.go +++ b/test/upgrade/postupgrade.go @@ -1,5 +1,3 @@ -// +build postupgrade - /* Copyright 2018 The Knative Authors @@ -25,6 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/pkg/ptr" ptest "knative.dev/pkg/test" + pkgupgrade "knative.dev/pkg/test/upgrade" v1 "knative.dev/serving/pkg/apis/serving/v1" serviceresourcenames "knative.dev/serving/pkg/reconciler/service/resources/names" rtesting "knative.dev/serving/pkg/testing/v1" @@ -33,9 +32,15 @@ import ( v1test "knative.dev/serving/test/v1" ) -func TestServicePostUpgrade(t *testing.T) { - t.Parallel() +// ServicePostUpgradeTest verifies an existing service after upgrade. +func ServicePostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePostUpgradeTest", func(c pkgupgrade.Context) { + servicePostUpgrade(c.T) + }) +} +func servicePostUpgrade(t *testing.T) { + t.Parallel() clients := e2e.Setup(t) // Before updating the service, the route and configuration objects should @@ -70,14 +75,22 @@ func routeHasGeneration(clients *test.Clients, serviceName string, generation in return routeObj.Generation == int64(generation), nil } -func TestServicePostUpgradeFromScaleToZero(t *testing.T) { - t.Parallel() - updateService(scaleToZeroServiceName, t) +// ServicePostUpgradeFromScaleToZeroTest verifies a scaled-to-zero service after upgrade. +func ServicePostUpgradeFromScaleToZeroTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("PostUpgradeFromScaleToZeroTest", func(c pkgupgrade.Context) { + updateService(scaleToZeroServiceName, c.T) + }) } -// TestBYORevisionPostUpgrade attempts to update the RouteSpec of a Service using BYO Revision name. This +// BYORevisionPostUpgradeTest attempts to update the RouteSpec of a Service using BYO Revision name. This // test is meant to catch new defaults that break the immutability of BYO Revision name. -func TestBYORevisionPostUpgrade(t *testing.T) { +func BYORevisionPostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("BYORevisionPostUpgradeTest", func(c pkgupgrade.Context) { + bYORevisionPostUpgrade(c.T) + }) +} + +func bYORevisionPostUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -136,12 +149,22 @@ func updateService(serviceName string, t *testing.T) { assertServiceResourcesUpdated(t, clients, names, routeURL, test.PizzaPlanetText2) } -func TestCreateNewServicePostUpgrade(t *testing.T) { - t.Parallel() - createNewService(postUpgradeServiceName, t) +// CreateNewServicePostUpgradeTest verifies creating a new service after upgrade. +func CreateNewServicePostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("CreateNewServicePostUpgradeTest", func(c pkgupgrade.Context) { + createNewService(postUpgradeServiceName, c.T) + }) +} + +// InitialScalePostUpgradeTest verifies that the service is ready after upgrade +// despite the fact that it does not receive any requests. +func InitialScalePostUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("InitialScalePostUpgradeTest", func(c pkgupgrade.Context) { + initialScalePostUpgrade(c.T) + }) } -func TestInitialScalePostUpgrade(t *testing.T) { +func initialScalePostUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) diff --git a/test/upgrade/service_preupgrade_test.go b/test/upgrade/preupgrade.go similarity index 68% rename from test/upgrade/service_preupgrade_test.go rename to test/upgrade/preupgrade.go index 03516d8cc9d4..5f6dcfec09d6 100644 --- a/test/upgrade/service_preupgrade_test.go +++ b/test/upgrade/preupgrade.go @@ -1,5 +1,3 @@ -// +build preupgrade - /* Copyright 2018 The Knative Authors @@ -21,6 +19,7 @@ package upgrade import ( "testing" + pkgupgrade "knative.dev/pkg/test/upgrade" "knative.dev/serving/pkg/apis/autoscaling" revisionresourcenames "knative.dev/serving/pkg/reconciler/revision/resources/names" rtesting "knative.dev/serving/pkg/testing/v1" @@ -29,7 +28,14 @@ import ( v1test "knative.dev/serving/test/v1" ) -func TestServicePreUpgrade(t *testing.T) { +// ServicePreUpgradeTest creates a service before upgrade. +func ServicePreUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePreUpgradeTest", func(c pkgupgrade.Context) { + servicePreUpgrade(c.T) + }) +} + +func servicePreUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) @@ -50,7 +56,15 @@ func TestServicePreUpgrade(t *testing.T) { assertServiceResourcesUpdated(t, clients, names, url, test.PizzaPlanetText1) } -func TestServicePreUpgradeAndScaleToZero(t *testing.T) { +// ServicePreUpgradeAndScaleToZeroTest creates a new service before +// upgrade and wait for it to scale to zero. +func ServicePreUpgradeAndScaleToZeroTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("ServicePreUpgradeAndScaleToZeroTest", func(c pkgupgrade.Context) { + servicePreUpgradeAndScaleToZero(c.T) + }) +} + +func servicePreUpgradeAndScaleToZero(t *testing.T) { t.Parallel() clients := e2e.Setup(t) @@ -75,9 +89,15 @@ func TestServicePreUpgradeAndScaleToZero(t *testing.T) { } } -// TestBYORevisionUpgrade creates a Service that uses the BYO Revision name functionality. This test +// BYORevisionPreUpgradeTest creates a Service that uses the BYO Revision name functionality. This test // is meant to catch new defaults that break bring your own revision name immutability. -func TestBYORevisionPreUpgrade(t *testing.T) { +func BYORevisionPreUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("BYORevisionPreUpgradeTest", func(c pkgupgrade.Context) { + bYORevisionPreUpgrade(c.T) + }) +} + +func bYORevisionPreUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -91,7 +111,15 @@ func TestBYORevisionPreUpgrade(t *testing.T) { } } -func TestInitialScalePreUpgrade(t *testing.T) { +// InitialScalePreUpgradeTest creates a service and lets it scale down to zero without +// sending any requests to it. +func InitialScalePreUpgradeTest() pkgupgrade.Operation { + return pkgupgrade.NewOperation("InitialScalePreUpgradeTest", func(c pkgupgrade.Context) { + initialScalePreUpgrade(c.T) + }) +} + +func initialScalePreUpgrade(t *testing.T) { t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ diff --git a/test/upgrade/probe.go b/test/upgrade/probe.go new file mode 100644 index 000000000000..38c4ebc31719 --- /dev/null +++ b/test/upgrade/probe.go @@ -0,0 +1,62 @@ +/* +Copyright 2019 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "context" + "flag" + + pkgupgrade "knative.dev/pkg/test/upgrade" + "knative.dev/serving/test" + "knative.dev/serving/test/e2e" + v1test "knative.dev/serving/test/v1" +) + +var successFraction = flag.Float64("probe.success_fraction", 1.0, "Fraction of probes required to pass during upgrade.") + +// ProbeTest sends requests to Knative services while performing an upgrade +// and verifies the expected success rate. +func ProbeTest() pkgupgrade.BackgroundOperation { + var clients *test.Clients + var names *test.ResourceNames + var prober test.Prober + return pkgupgrade.NewBackgroundVerification("ProbeTest", + func(c pkgupgrade.Context) { + // Setup + clients = e2e.Setup(c.T) + names = &test.ResourceNames{ + Service: "upgrade-probe", + Image: test.PizzaPlanet1, + } + objects, err := v1test.CreateServiceReady(c.T, clients, names) + if err != nil { + c.T.Fatal("Failed to create Service:", err) + } + url := objects.Service.Status.URL.URL() + + // This polls until we get a 200 with the right body. + assertServiceResourcesUpdated(c.T, clients, *names, url, test.PizzaPlanetText1) + + prober = test.RunRouteProber(c.T.Logf, clients, url, test.AddRootCAtoTransport(context.Background(), c.T.Logf, clients, test.ServingFlags.HTTPS)) + }, + func(c pkgupgrade.Context) { + // Verify + test.EnsureTearDown(c.T, clients, names) + test.AssertProberSLO(c.T, prober, *successFraction) + }, + ) +} diff --git a/test/upgrade/probe_test.go b/test/upgrade/probe_test.go deleted file mode 100644 index 981ee29579e7..000000000000 --- a/test/upgrade/probe_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// +build probe - -/* -Copyright 2019 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package upgrade - -import ( - "context" - "flag" - "io/ioutil" - "testing" - - "knative.dev/serving/test" - "knative.dev/serving/test/e2e" - v1test "knative.dev/serving/test/v1" -) - -const probePipe = "/tmp/prober-signal" - -var successFraction = flag.Float64("probe.success_fraction", 1.0, "Fraction of probes required to pass during upgrade.") - -func TestProbe(t *testing.T) { - t.Parallel() - // We run the prober as a golang test because it fits in nicely with - // the rest of our integration tests, and AssertProberDefault needs - // a *testing.T. Unfortunately, "go test" intercepts signals, so we - // can't coordinate with the test by just sending e.g. SIGCONT, so we - // create a named pipe and wait for the upgrade script to write to it - // to signal that we should stop probing. - createPipe(t, probePipe) - - clients := e2e.Setup(t) - names := test.ResourceNames{ - Service: "upgrade-probe", - Image: test.PizzaPlanet1, - } - - test.EnsureTearDown(t, clients, &names) - - objects, err := v1test.CreateServiceReady(t, clients, &names) - if err != nil { - t.Fatal("Failed to create Service:", err) - } - url := objects.Service.Status.URL.URL() - - // This polls until we get a 200 with the right body. - assertServiceResourcesUpdated(t, clients, names, url, test.PizzaPlanetText1) - - prober := test.RunRouteProber(t.Logf, clients, url, test.AddRootCAtoTransport(context.Background(), t.Logf, clients, test.ServingFlags.HTTPS)) - defer test.CheckSLO(*successFraction, t.Name(), prober) - - // e2e-upgrade-test.sh will close this pipe to signal the upgrade is - // over, at which point we will finish the test and check the prober. - ioutil.ReadFile(probePipe) -} diff --git a/test/upgrade/upgrade.go b/test/upgrade/upgrade.go index edd135a3c63c..e91cfa02358f 100644 --- a/test/upgrade/upgrade.go +++ b/test/upgrade/upgrade.go @@ -18,11 +18,8 @@ package upgrade import ( "context" - "errors" "fmt" "net/url" - "os" - "syscall" "testing" // Mysteriously required to support GCP auth (required by k8s libs). @@ -66,6 +63,7 @@ func assertServiceResourcesUpdated(t testing.TB, clients *test.Clients, names te } func createNewService(serviceName string, t *testing.T) { + t.Parallel() clients := e2e.Setup(t) names := test.ResourceNames{ @@ -80,17 +78,3 @@ func createNewService(serviceName string, t *testing.T) { url := resources.Service.Status.URL.URL() assertServiceResourcesUpdated(t, clients, names, url, test.PizzaPlanetText1) } - -// createPipe create a named pipe. It fails the test if any error except -// already exist happens. -func createPipe(t *testing.T, name string) { - if err := syscall.Mkfifo(name, 0666); err != nil { - if !errors.Is(err, os.ErrExist) { - t.Fatal("Failed to create pipe:", err) - } - } - - test.EnsureCleanup(t, func() { - os.Remove(name) - }) -} diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go new file mode 100644 index 000000000000..0b1debfbb730 --- /dev/null +++ b/test/upgrade/upgrade_test.go @@ -0,0 +1,79 @@ +// +build upgrade + +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package upgrade + +import ( + "testing" + + "go.uber.org/zap" + _ "knative.dev/pkg/system/testing" + pkgupgrade "knative.dev/pkg/test/upgrade" + "knative.dev/serving/test/upgrade/installation" +) + +func TestServingUpgrades(t *testing.T) { + c := newUpgradeConfig(t) + suite := pkgupgrade.Suite{ + Tests: pkgupgrade.Tests{ + PreUpgrade: []pkgupgrade.Operation{ + ServicePreUpgradeTest(), + ServicePreUpgradeAndScaleToZeroTest(), + BYORevisionPreUpgradeTest(), + InitialScalePreUpgradeTest(), + }, + PostUpgrade: []pkgupgrade.Operation{ + ServicePostUpgradeTest(), + ServicePostUpgradeFromScaleToZeroTest(), + BYORevisionPostUpgradeTest(), + CreateNewServicePostUpgradeTest(), + InitialScalePostUpgradeTest(), + }, + PostDowngrade: []pkgupgrade.Operation{ + ServicePostDowngradeTest(), + CreateNewServicePostDowngradeTest(), + }, + Continual: []pkgupgrade.BackgroundOperation{ + ProbeTest(), + AutoscaleSustainingTest(c.Log), + AutoscaleSustainingWithTBCTest(c.Log), + }, + }, + Installations: pkgupgrade.Installations{ + Base: []pkgupgrade.Operation{ + // Do nothing. The initial version is already installed by scripts + // together with additional test resources. + }, + UpgradeWith: []pkgupgrade.Operation{ + installation.Head(), + }, + DowngradeWith: []pkgupgrade.Operation{ + installation.LatestRelease(), + }, + }, + } + suite.Execute(c) +} + +func newUpgradeConfig(t *testing.T) pkgupgrade.Configuration { + log, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + return pkgupgrade.Configuration{T: t, Log: log} +} diff --git a/third_party/VENDOR-LICENSE/knative.dev/hack/shell/LICENSE b/third_party/VENDOR-LICENSE/knative.dev/hack/shell/LICENSE new file mode 100644 index 000000000000..261eeb9e9f8b --- /dev/null +++ b/third_party/VENDOR-LICENSE/knative.dev/hack/shell/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/knative.dev/hack/shell/executor.go b/vendor/knative.dev/hack/shell/executor.go new file mode 100644 index 000000000000..e6308a0706fd --- /dev/null +++ b/vendor/knative.dev/hack/shell/executor.go @@ -0,0 +1,189 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "strings" + "time" +) + +const ( + defaultLabelOut = "[OUT]" + defaultLabelErr = "[ERR]" + executeMode = 0700 +) + +// ErrNoProjectLocation is returned if user didnt provided the project location. +var ErrNoProjectLocation = errors.New("project location isn't provided") + +// NewExecutor creates a new executor from given config. +func NewExecutor(config ExecutorConfig) Executor { + configureDefaultValues(&config) + return &streamingExecutor{ + ExecutorConfig: config, + } +} + +// RunScript executes a shell script with args. +func (s *streamingExecutor) RunScript(script Script, args ...string) error { + err := validate(s.ExecutorConfig) + if err != nil { + return err + } + cnt := script.scriptContent(s.ProjectLocation, args) + return withTempScript(cnt, func(bin string) error { + return stream(bin, s.ExecutorConfig, script.Label) + }) +} + +// RunFunction executes a shell function with args. +func (s *streamingExecutor) RunFunction(fn Function, args ...string) error { + err := validate(s.ExecutorConfig) + if err != nil { + return err + } + cnt := fn.scriptContent(s.ProjectLocation, args) + return withTempScript(cnt, func(bin string) error { + return stream(bin, s.ExecutorConfig, fn.Label) + }) +} + +type streamingExecutor struct { + ExecutorConfig +} + +func validate(config ExecutorConfig) error { + if config.ProjectLocation == nil { + return ErrNoProjectLocation + } + return nil +} + +func configureDefaultValues(config *ExecutorConfig) { + if config.Out == nil { + config.Out = os.Stdout + } + if config.Err == nil { + config.Err = os.Stderr + } + if config.LabelOut == "" { + config.LabelOut = defaultLabelOut + } + if config.LabelErr == "" { + config.LabelErr = defaultLabelErr + } + if config.Environ == nil { + config.Environ = os.Environ() + } + if !config.SkipDate && config.DateFormat == "" { + config.DateFormat = time.StampMilli + } + if config.PrefixFunc == nil { + config.PrefixFunc = defaultPrefixFunc + } +} + +func stream(bin string, cfg ExecutorConfig, label string) error { + c := exec.Command(bin) + c.Env = cfg.Environ + c.Stdout = NewPrefixer(cfg.Out, prefixFunc(StreamTypeOut, label, cfg)) + c.Stderr = NewPrefixer(cfg.Err, prefixFunc(StreamTypeErr, label, cfg)) + return c.Run() +} + +func prefixFunc(st StreamType, label string, cfg ExecutorConfig) func() string { + return func() string { + return cfg.PrefixFunc(st, label, cfg) + } +} + +func defaultPrefixFunc(st StreamType, label string, cfg ExecutorConfig) string { + sep := " " + var buf []string + if !cfg.SkipDate { + dt := time.Now().Format(cfg.DateFormat) + buf = append(buf, dt) + } + buf = append(buf, label) + switch st { + case StreamTypeOut: + buf = append(buf, cfg.LabelOut) + case StreamTypeErr: + buf = append(buf, cfg.LabelErr) + } + return strings.Join(buf, sep) + sep +} + +func withTempScript(contents string, fn func(bin string) error) error { + tmpfile, err := ioutil.TempFile("", "shellout-*.sh") + if err != nil { + return err + } + _, err = tmpfile.WriteString(contents) + if err != nil { + return err + } + err = tmpfile.Chmod(executeMode) + if err != nil { + return err + } + err = tmpfile.Close() + if err != nil { + return err + } + defer func() { + // clean up + _ = os.Remove(tmpfile.Name()) + }() + + return fn(tmpfile.Name()) +} + +func (fn *Function) scriptContent(location ProjectLocation, args []string) string { + return fmt.Sprintf(`#!/usr/bin/env bash + +set -Eeuo pipefail + +cd "%s" +source %s + +%s %s +`, location.RootPath(), fn.ScriptPath, fn.FunctionName, quoteArgs(args)) +} + +func (sc *Script) scriptContent(location ProjectLocation, args []string) string { + return fmt.Sprintf(`#!/usr/bin/env bash + +set -Eeuo pipefail + +cd "%s" +%s %s +`, location.RootPath(), sc.ScriptPath, quoteArgs(args)) +} + +func quoteArgs(args []string) string { + quoted := make([]string, len(args)) + for i, arg := range args { + quoted[i] = "\"" + strings.ReplaceAll(arg, "\"", "\\\"") + "\"" + } + return strings.Join(quoted, " ") +} diff --git a/vendor/knative.dev/hack/shell/fail-example.sh b/vendor/knative.dev/hack/shell/fail-example.sh new file mode 100644 index 000000000000..551ce662a340 --- /dev/null +++ b/vendor/knative.dev/hack/shell/fail-example.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "$*" >&2 diff --git a/vendor/knative.dev/hack/shell/prefixer.go b/vendor/knative.dev/hack/shell/prefixer.go new file mode 100644 index 000000000000..273ee1cc4fff --- /dev/null +++ b/vendor/knative.dev/hack/shell/prefixer.go @@ -0,0 +1,68 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "bytes" + "io" +) + +// NewPrefixer creates a new prefixer that forwards all calls to Write() to +// writer.Write() with all lines prefixed with the value of prefix. Having a +// function instead of a static prefix allows to print timestamps or other +// changing information. +func NewPrefixer(writer io.Writer, prefix func() string) io.Writer { + return &prefixer{prefix: prefix, writer: writer, trailingNewline: true} +} + +type prefixer struct { + prefix func() string + writer io.Writer + trailingNewline bool + buf bytes.Buffer // reuse buffer to save allocations +} + +func (pf *prefixer) Write(payload []byte) (int, error) { + pf.buf.Reset() // clear the buffer + + for _, b := range payload { + if pf.trailingNewline { + pf.buf.WriteString(pf.prefix()) + pf.trailingNewline = false + } + + pf.buf.WriteByte(b) + + if b == '\n' { + // do not print the prefix right after the newline character as this might + // be the very last character of the stream and we want to avoid a trailing prefix. + pf.trailingNewline = true + } + } + + n, err := pf.writer.Write(pf.buf.Bytes()) + if err != nil { + // never return more than original length to satisfy io.Writer interface + if n > len(payload) { + n = len(payload) + } + return n, err + } + + // return original length to satisfy io.Writer interface + return len(payload), nil +} diff --git a/vendor/knative.dev/hack/shell/project.go b/vendor/knative.dev/hack/shell/project.go new file mode 100644 index 000000000000..2d7ba9738b08 --- /dev/null +++ b/vendor/knative.dev/hack/shell/project.go @@ -0,0 +1,52 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import ( + "errors" + "path" + "runtime" +) + +// ErrCantGetCaller is raised when we can't calculate a caller of NewProjectLocation. +var ErrCantGetCaller = errors.New("can't get caller") + +// NewProjectLocation creates a ProjectLocation that is used to calculate +// relative paths within the project. +func NewProjectLocation(pathToRoot string) (ProjectLocation, error) { + _, filename, _, ok := runtime.Caller(1) + if !ok { + return nil, ErrCantGetCaller + } + return &callerLocation{ + caller: filename, + pathToRoot: pathToRoot, + }, nil +} + +// RootPath return a path to root of the project. +func (c *callerLocation) RootPath() string { + return path.Join(path.Dir(c.caller), c.pathToRoot) +} + +// callerLocation holds a caller Go file, and a relative location to a project +// root directory. This information can be used to calculate relative paths and +// properly source shell scripts. +type callerLocation struct { + caller string + pathToRoot string +} diff --git a/vendor/knative.dev/hack/shell/types.go b/vendor/knative.dev/hack/shell/types.go new file mode 100644 index 000000000000..8e34515206dd --- /dev/null +++ b/vendor/knative.dev/hack/shell/types.go @@ -0,0 +1,82 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package shell + +import "io" + +// ProjectLocation represents a project location on a file system. +type ProjectLocation interface { + RootPath() string +} + +// Script represents a script to be executed. +type Script struct { + Label string + ScriptPath string +} + +// Function represents a function, whom will be sourced from Script file, +// and executed. +type Function struct { + Script + FunctionName string +} + +// ExecutorConfig holds a executor configuration options. +type ExecutorConfig struct { + ProjectLocation + Streams + Labels + Environ []string +} + +// StreamType represets either output or error stream. +type StreamType int + +const ( + // StreamTypeOut represents process output stream. + StreamTypeOut StreamType = iota + // StreamTypeErr represents process error stream. + StreamTypeErr +) + +// PrefixFunc is used to build a prefix that will be added to each line of the +// script/function output or error stream. +type PrefixFunc func(st StreamType, label string, config ExecutorConfig) string + +// Labels holds a labels to be used to prefix Out and Err streams of executed +// shells scripts/functions. +type Labels struct { + LabelOut string + LabelErr string + SkipDate bool + DateFormat string + PrefixFunc +} + +// Streams holds a streams of a shell scripts/functions. +type Streams struct { + Out io.Writer + Err io.Writer +} + +// Executor represents a executor that can execute shell scripts and call +// functions directly. +type Executor interface { + RunScript(script Script, args ...string) error + RunFunction(fn Function, args ...string) error +} diff --git a/vendor/knative.dev/pkg/test/upgrade/functions.go b/vendor/knative.dev/pkg/test/upgrade/functions.go new file mode 100644 index 000000000000..50dc637fefa4 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/functions.go @@ -0,0 +1,162 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "time" + + "go.uber.org/zap" +) + +// Execute the Suite of upgrade tests with a Configuration given. +func (s *Suite) Execute(c Configuration) { + l := c.logger() + se := suiteExecution{ + suite: enrichSuite(s), + configuration: c, + failed: false, + logger: l, + } + l.Info("🏃 Running upgrade test suite...") + + se.execute() + + if !se.failed { + l.Info("🥳🎉 Success! Upgrade suite completed without errors.") + } else { + l.Error("💣🤬💔️ Upgrade suite have failed!") + } +} + +// NewOperation creates a new upgrade operation or test. +func NewOperation(name string, handler func(c Context)) Operation { + return &simpleOperation{name: name, handler: handler} +} + +// NewBackgroundVerification is convenience function to easily setup a +// background operation that will setup environment and then verify environment +// status after receiving a StopEvent. +func NewBackgroundVerification(name string, setup func(c Context), verify func(c Context)) BackgroundOperation { + return NewBackgroundOperation(name, setup, func(bc BackgroundContext) { + WaitForStopEvent(bc, WaitForStopEventConfiguration{ + Name: name, + OnStop: func(event StopEvent) { + verify(Context{ + T: event.T, + Log: bc.Log, + }) + }, + OnWait: DefaultOnWait, + WaitTime: DefaultWaitTime, + }) + }) +} + +// NewBackgroundOperation creates a new background operation or test that can be +// notified to stop its operation. +func NewBackgroundOperation(name string, setup func(c Context), + handler func(bc BackgroundContext)) BackgroundOperation { + return &simpleBackgroundOperation{ + name: name, + setup: setup, + handler: handler, + } +} + +// WaitForStopEvent will wait until upgrade suite sends a stop event to it. +// After that happen a handler is invoked to verify environment state and report +// failures. +func WaitForStopEvent(bc BackgroundContext, w WaitForStopEventConfiguration) { + for { + select { + case stopEvent := <-bc.Stop: + handleStopEvent(stopEvent, bc, w) + return + default: + w.OnWait(bc, w) + } + time.Sleep(w.WaitTime) + } +} + +func (c Configuration) logger() *zap.SugaredLogger { + return c.Log.Sugar() +} + +// Name returns a friendly human readable text. +func (s *StopEvent) Name() string { + return s.name +} + +func handleStopEvent( + se StopEvent, + bc BackgroundContext, + wc WaitForStopEventConfiguration, +) { + bc.Log.Infof("%s have received a stop event: %s", wc.Name, se.Name()) + defer close(se.Finished) + wc.OnStop(se) +} + +func enrichSuite(s *Suite) *enrichedSuite { + es := &enrichedSuite{ + installations: s.Installations, + tests: enrichedTests{ + preUpgrade: s.Tests.PreUpgrade, + postUpgrade: s.Tests.PostUpgrade, + postDowngrade: s.Tests.PostDowngrade, + continual: make([]stoppableOperation, len(s.Tests.Continual)), + }, + } + for i, test := range s.Tests.Continual { + es.tests.continual[i] = stoppableOperation{ + BackgroundOperation: test, + stop: make(chan StopEvent), + } + } + return es +} + +// Name is a human readable operation title, and it will be used in t.Run. +func (h *simpleOperation) Name() string { + return h.name +} + +// Handler is a function that will be called to perform an operation. +func (h *simpleOperation) Handler() func(c Context) { + return h.handler +} + +// Name is a human readable operation title, and it will be used in t.Run. +func (s *simpleBackgroundOperation) Name() string { + return s.name +} + +// Setup method may be used to set up environment before upgrade/downgrade is +// performed. +func (s *simpleBackgroundOperation) Setup() func(c Context) { + return s.setup +} + +// Handler will be executed in background while upgrade/downgrade is being +// executed. It can be used to constantly validate environment during that +// time and/or wait for StopEvent being sent. After StopEvent is received +// user should validate environment, clean up resources, and report found +// issues to testing.T forwarded in StepEvent. +func (s *simpleBackgroundOperation) Handler() func(bc BackgroundContext) { + return s.handler +} diff --git a/vendor/knative.dev/pkg/test/upgrade/private_types.go b/vendor/knative.dev/pkg/test/upgrade/private_types.go new file mode 100644 index 000000000000..e34f090050f1 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/private_types.go @@ -0,0 +1,63 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import "go.uber.org/zap" + +type suiteExecution struct { + suite *enrichedSuite + configuration Configuration + failed bool + logger *zap.SugaredLogger +} + +type enrichedSuite struct { + installations Installations + tests enrichedTests +} + +type enrichedTests struct { + preUpgrade []Operation + postUpgrade []Operation + postDowngrade []Operation + continual []stoppableOperation +} + +type stoppableOperation struct { + BackgroundOperation + stop chan StopEvent +} + +type operationGroup struct { + num int + operations []Operation + groupName string + groupTemplate string + elementTemplate string + skippingGroupTemplate string +} + +type simpleOperation struct { + name string + handler func(c Context) +} + +type simpleBackgroundOperation struct { + name string + setup func(c Context) + handler func(bc BackgroundContext) +} diff --git a/vendor/knative.dev/pkg/test/upgrade/steps.go b/vendor/knative.dev/pkg/test/upgrade/steps.go new file mode 100644 index 000000000000..c50bdcfb6272 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/steps.go @@ -0,0 +1,155 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "testing" +) + +const skippingOperationTemplate = `Skipping "%s" as previous operation have failed` + +func (se *suiteExecution) installingBase(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.Base, + groupName: "InstallingBase", + elementTemplate: `%d.%d) Installing base install of "%s".`, + skippingGroupTemplate: "%d) 💿 No base installation registered. Skipping.", + groupTemplate: "%d) 💿 Installing base installations. %d are registered.", + }) +} + +func (se *suiteExecution) preUpgradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.preUpgrade, + groupName: "PreUpgradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No pre upgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality before upgrade is performed." + + " %d tests are registered.", + }) +} + +func (se *suiteExecution) startContinualTests(num int) { + l := se.logger + operations := se.suite.tests.continual + groupTemplate := "%d) 🔄 Starting continual tests. " + + "%d tests are registered." + elementTemplate := `%d.%d) Starting continual tests of "%s".` + numOps := len(operations) + se.configuration.T.Run("ContinualTests", func(t *testing.T) { + if numOps > 0 { + l.Infof(groupTemplate, num, numOps) + for i := range operations { + operation := operations[i] + l.Infof(elementTemplate, num, i+1, operation.Name()) + if se.failed { + l.Debugf(skippingOperationTemplate, operation.Name()) + return + } + setup := operation.Setup() + t.Run("Setup"+operation.Name(), func(t *testing.T) { + setup(Context{T: t, Log: l}) + }) + handler := operation.Handler() + go func() { + bc := BackgroundContext{Log: l, Stop: operation.stop} + handler(bc) + }() + + se.failed = se.failed || t.Failed() + if se.failed { + return + } + } + + } else { + l.Infof("%d) 🔄 No continual tests registered. Skipping.", num) + } + }) +} + +func (se *suiteExecution) verifyContinualTests(num int) { + l := se.logger + testsCount := len(se.suite.tests.continual) + if testsCount > 0 { + se.configuration.T.Run("VerifyContinualTests", func(t *testing.T) { + l.Infof("%d) ✋ Verifying %d running continual tests.", num, testsCount) + for i, operation := range se.suite.tests.continual { + t.Run(operation.Name(), func(t *testing.T) { + l.Infof(`%d.%d) Verifying "%s".`, num, i+1, operation.Name()) + finished := make(chan struct{}) + operation.stop <- StopEvent{ + T: t, + Finished: finished, + name: "Stop of " + operation.Name(), + } + <-finished + se.failed = se.failed || t.Failed() + l.Debugf(`Finished "%s"`, operation.Name()) + }) + } + }) + } +} + +func (se *suiteExecution) upgradeWith(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.UpgradeWith, + groupName: "UpgradeWith", + elementTemplate: `%d.%d) Upgrading with "%s".`, + skippingGroupTemplate: "%d) 📀 No upgrade operations registered. Skipping.", + groupTemplate: "%d) 📀 Upgrading with %d registered operations.", + }) +} + +func (se *suiteExecution) postUpgradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.postUpgrade, + groupName: "PostUpgradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No post upgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality after upgrade is performed." + + " %d tests are registered.", + }) +} + +func (se *suiteExecution) downgradeWith(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.installations.DowngradeWith, + groupName: "DowngradeWith", + elementTemplate: `%d.%d) Downgrading with "%s".`, + skippingGroupTemplate: "%d) 💿 No downgrade operations registered. Skipping.", + groupTemplate: "%d) 💿 Downgrading with %d registered operations.", + }) +} + +func (se *suiteExecution) postDowngradeTests(num int) { + se.processOperationGroup(operationGroup{ + num: num, + operations: se.suite.tests.postDowngrade, + groupName: "PostDowngradeTests", + elementTemplate: `%d.%d) Testing with "%s".`, + skippingGroupTemplate: "%d) ✅️️ No post downgrade tests registered. Skipping.", + groupTemplate: "%d) ✅️️ Testing functionality after downgrade is performed." + + " %d tests are registered.", + }) +} diff --git a/vendor/knative.dev/pkg/test/upgrade/suite_execution.go b/vendor/knative.dev/pkg/test/upgrade/suite_execution.go new file mode 100644 index 000000000000..70c5ea095ea7 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/suite_execution.go @@ -0,0 +1,85 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "testing" +) + +func (se *suiteExecution) processOperationGroup(op operationGroup) { + l := se.logger + se.configuration.T.Run(op.groupName, func(t *testing.T) { + if len(op.operations) > 0 { + l.Infof(op.groupTemplate, op.num, len(op.operations)) + for i, operation := range op.operations { + l.Infof(op.elementTemplate, op.num, i+1, operation.Name()) + if se.failed { + l.Debugf(skippingOperationTemplate, operation.Name()) + return + } + handler := operation.Handler() + t.Run(operation.Name(), func(t *testing.T) { + handler(Context{T: t, Log: l}) + }) + se.failed = se.failed || t.Failed() + if se.failed { + return + } + } + } else { + l.Infof(op.skippingGroupTemplate, op.num) + } + }) +} + +func (se *suiteExecution) execute() { + idx := 1 + operations := []func(num int){ + se.installingBase, + se.preUpgradeTests, + } + for _, operation := range operations { + operation(idx) + idx++ + if se.failed { + return + } + } + + se.startContinualTests(idx) + idx++ + if se.failed { + return + } + defer func() { + se.verifyContinualTests(idx) + }() + + operations = []func(num int){ + se.upgradeWith, + se.postUpgradeTests, + se.downgradeWith, + se.postDowngradeTests, + } + for _, operation := range operations { + operation(idx) + idx++ + if se.failed { + return + } + } +} diff --git a/vendor/knative.dev/pkg/test/upgrade/types.go b/vendor/knative.dev/pkg/test/upgrade/types.go new file mode 100644 index 000000000000..afcccec332a7 --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/types.go @@ -0,0 +1,123 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import ( + "testing" + "time" + + "go.uber.org/zap" +) + +// Suite represents a upgrade tests suite that can be executed and will perform +// execution in predictable manner. +type Suite struct { + Tests Tests + Installations Installations +} + +// Tests holds a list of operations for various part of upgrade suite. +type Tests struct { + PreUpgrade []Operation + PostUpgrade []Operation + PostDowngrade []Operation + Continual []BackgroundOperation +} + +// Installations holds a list of operations that will install Knative components +// in different versions. +type Installations struct { + Base []Operation + UpgradeWith []Operation + DowngradeWith []Operation +} + +// Operation represents a upgrade test operation like test or installation that +// can be provided by specific component or reused in aggregating components. +type Operation interface { + // Name is a human readable operation title, and it will be used in t.Run. + Name() string + // Handler is a function that will be called to perform an operation. + Handler() func(c Context) +} + +// BackgroundOperation represents a upgrade test operation that will be +// performed in background while other operations is running. To achieve that +// a passed BackgroundContext should be used to synchronize it's operations with +// Ready and Stop channels. +type BackgroundOperation interface { + // Name is a human readable operation title, and it will be used in t.Run. + Name() string + // Setup method may be used to set up environment before upgrade/downgrade is + // performed. + Setup() func(c Context) + // Handler will be executed in background while upgrade/downgrade is being + // executed. It can be used to constantly validate environment during that + // time and/or wait for StopEvent being sent. After StopEvent is received + // user should validate environment, clean up resources, and report found + // issues to testing.T forwarded in StepEvent. + Handler() func(bc BackgroundContext) +} + +// Context is an object that is passed to every operation. It contains testing.T +// for error reporting and zap.SugaredLogger for unbuffered logging. +type Context struct { + T *testing.T + Log *zap.SugaredLogger +} + +// BackgroundContext is a upgrade test execution context that will be passed +// down to each handler of BackgroundOperation. It contains a StopEvent channel +// which end user should use to obtain a testing.T for error reporting. Until +// StopEvent is sent user may use zap.SugaredLogger to log state of execution if +// necessary. +type BackgroundContext struct { + Log *zap.SugaredLogger + Stop <-chan StopEvent +} + +// StopEvent represents an event that is to be received by background operation +// to indicate that is should stop it's operations and validate results using +// passed T. User should use Finished channel to signalize upgrade suite that +// all stop & verify operations are finished and it is safe to end tests. +type StopEvent struct { + T *testing.T + Finished chan<- struct{} + name string +} + +// WaitForStopEventConfiguration holds a values to be used be WaitForStopEvent +// function. OnStop will be called when StopEvent is sent. OnWait will be +// invoked in a loop while waiting, and each wait act is driven by WaitTime +// amount. +type WaitForStopEventConfiguration struct { + Name string + OnStop func(event StopEvent) + OnWait func(bc BackgroundContext, self WaitForStopEventConfiguration) + WaitTime time.Duration +} + +// Configuration holds required and optional configuration to run upgrade tests. +type Configuration struct { + T *testing.T + Log *zap.Logger +} + +// SuiteExecutor is to execute upgrade test suite. +type SuiteExecutor interface { + Execute(c Configuration) +} diff --git a/vendor/knative.dev/pkg/test/upgrade/vars.go b/vendor/knative.dev/pkg/test/upgrade/vars.go new file mode 100644 index 000000000000..8a8ba93e11ae --- /dev/null +++ b/vendor/knative.dev/pkg/test/upgrade/vars.go @@ -0,0 +1,32 @@ +/* + * Copyright 2020 The Knative Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package upgrade + +import "time" + +var ( + // DefaultWaitTime holds a default value for WaitForStopEventConfiguration + // when used within a NewBackgroundVerification function. + DefaultWaitTime = 20 * time.Millisecond + + // DefaultOnWait is a implementation that will be called by default for each + // wait performed by WaitForStopEvent when used within + // NewBackgroundVerification function. + DefaultOnWait = func(bc BackgroundContext, self WaitForStopEventConfiguration) { + // do nothing by default + } +) diff --git a/vendor/modules.txt b/vendor/modules.txt index a47bb01de1b8..eb47c42ae49e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1117,6 +1117,7 @@ knative.dev/pkg/test/mako/config knative.dev/pkg/test/monitoring knative.dev/pkg/test/slackutil knative.dev/pkg/test/spoof +knative.dev/pkg/test/upgrade knative.dev/pkg/test/vegeta/pacers knative.dev/pkg/test/zipkin knative.dev/pkg/tracing From 0c652e1f84c4e2b8d30e2c00366acb994a8a0fd6 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 1 Dec 2020 14:52:55 +0100 Subject: [PATCH 02/18] Use umbrella functions for individual groups of tests That's for easy reuse in other repos such as knative/operator --- test/upgrade/autoscaler.go | 10 ++++------ test/upgrade/continual.go | 11 +++++++++++ test/upgrade/postdowngrade.go | 7 +++++++ test/upgrade/postupgrade.go | 10 ++++++++++ test/upgrade/preupgrade.go | 12 ++++++++++-- test/upgrade/upgrade_test.go | 26 ++++---------------------- 6 files changed, 46 insertions(+), 30 deletions(-) create mode 100644 test/upgrade/continual.go diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go index 58a8b4c7b511..cdc4c58de79e 100644 --- a/test/upgrade/autoscaler.go +++ b/test/upgrade/autoscaler.go @@ -19,8 +19,6 @@ package upgrade import ( "time" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" "knative.dev/serving/test" @@ -39,7 +37,7 @@ const ( // AutoscaleSustainingTest checks that when traffic increases a knative app // scales up and sustains the scale as long as the traffic sustains, despite whether // it is switching modes between normal and panic. -func AutoscaleSustainingTest(logger *zap.Logger) pkgupgrade.BackgroundOperation { +func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { var ctx *e2e.TestContext var grp errgroup.Group stopCh := make(chan time.Time) @@ -51,7 +49,7 @@ func AutoscaleSustainingTest(logger *zap.Logger) pkgupgrade.BackgroundOperation autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. })) grp.Go(func() error { - return e2e.AssertAutoscaleUpToNumPods(ctx, logger.Sugar().Infof, 1, 10, stopCh, false /* quick */) + return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, 1, 10, stopCh, false /* quick */) }) }, func(c pkgupgrade.Context) { @@ -68,7 +66,7 @@ func AutoscaleSustainingTest(logger *zap.Logger) pkgupgrade.BackgroundOperation // AutoscaleSustainingWithTBCTest checks that when traffic increases and the activator is // in the path a knative app scales up and sustains the scale. -func AutoscaleSustainingWithTBCTest(logger *zap.Logger) pkgupgrade.BackgroundOperation { +func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { var ctx *e2e.TestContext var grp errgroup.Group stopCh := make(chan time.Time) @@ -80,7 +78,7 @@ func AutoscaleSustainingWithTBCTest(logger *zap.Logger) pkgupgrade.BackgroundOpe autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. })) grp.Go(func() error { - return e2e.AssertAutoscaleUpToNumPods(ctx, logger.Sugar().Infof, 1, 10, stopCh, false /* quick */) + return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, 1, 10, stopCh, false /* quick */) }) }, func(c pkgupgrade.Context) { diff --git a/test/upgrade/continual.go b/test/upgrade/continual.go new file mode 100644 index 000000000000..397503f0b21b --- /dev/null +++ b/test/upgrade/continual.go @@ -0,0 +1,11 @@ +package upgrade + +import pkgupgrade "knative.dev/pkg/test/upgrade" + +func ContinualTests() []pkgupgrade.BackgroundOperation { + return []pkgupgrade.BackgroundOperation{ + ProbeTest(), + AutoscaleSustainingTest(), + AutoscaleSustainingWithTBCTest(), + } +} diff --git a/test/upgrade/postdowngrade.go b/test/upgrade/postdowngrade.go index d663b33bd6de..ec2a08179549 100644 --- a/test/upgrade/postdowngrade.go +++ b/test/upgrade/postdowngrade.go @@ -31,6 +31,13 @@ import ( v1test "knative.dev/serving/test/v1" ) +func ServingPostDowngradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + ServicePostDowngradeTest(), + CreateNewServicePostDowngradeTest(), + } +} + // ServicePostDowngradeTest verifies an existing service after downgrade. func ServicePostDowngradeTest() pkgupgrade.Operation { return pkgupgrade.NewOperation("ServicePostDowngradeTest", func(c pkgupgrade.Context) { diff --git a/test/upgrade/postupgrade.go b/test/upgrade/postupgrade.go index e32bd61b173b..a7a073c42cc1 100644 --- a/test/upgrade/postupgrade.go +++ b/test/upgrade/postupgrade.go @@ -32,6 +32,16 @@ import ( v1test "knative.dev/serving/test/v1" ) +func ServingPostUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + ServicePostUpgradeTest(), + ServicePostUpgradeFromScaleToZeroTest(), + BYORevisionPostUpgradeTest(), + CreateNewServicePostUpgradeTest(), + InitialScalePostUpgradeTest(), + } +} + // ServicePostUpgradeTest verifies an existing service after upgrade. func ServicePostUpgradeTest() pkgupgrade.Operation { return pkgupgrade.NewOperation("ServicePostUpgradeTest", func(c pkgupgrade.Context) { diff --git a/test/upgrade/preupgrade.go b/test/upgrade/preupgrade.go index 5f6dcfec09d6..c0f635439942 100644 --- a/test/upgrade/preupgrade.go +++ b/test/upgrade/preupgrade.go @@ -17,8 +17,6 @@ limitations under the License. package upgrade import ( - "testing" - pkgupgrade "knative.dev/pkg/test/upgrade" "knative.dev/serving/pkg/apis/autoscaling" revisionresourcenames "knative.dev/serving/pkg/reconciler/revision/resources/names" @@ -26,8 +24,18 @@ import ( "knative.dev/serving/test" "knative.dev/serving/test/e2e" v1test "knative.dev/serving/test/v1" + "testing" ) +func ServingPreUpgradeTests() []pkgupgrade.Operation { + return []pkgupgrade.Operation{ + ServicePreUpgradeTest(), + ServicePreUpgradeAndScaleToZeroTest(), + BYORevisionPreUpgradeTest(), + InitialScalePreUpgradeTest(), + } +} + // ServicePreUpgradeTest creates a service before upgrade. func ServicePreUpgradeTest() pkgupgrade.Operation { return pkgupgrade.NewOperation("ServicePreUpgradeTest", func(c pkgupgrade.Context) { diff --git a/test/upgrade/upgrade_test.go b/test/upgrade/upgrade_test.go index 0b1debfbb730..658bd3a5626a 100644 --- a/test/upgrade/upgrade_test.go +++ b/test/upgrade/upgrade_test.go @@ -31,28 +31,10 @@ func TestServingUpgrades(t *testing.T) { c := newUpgradeConfig(t) suite := pkgupgrade.Suite{ Tests: pkgupgrade.Tests{ - PreUpgrade: []pkgupgrade.Operation{ - ServicePreUpgradeTest(), - ServicePreUpgradeAndScaleToZeroTest(), - BYORevisionPreUpgradeTest(), - InitialScalePreUpgradeTest(), - }, - PostUpgrade: []pkgupgrade.Operation{ - ServicePostUpgradeTest(), - ServicePostUpgradeFromScaleToZeroTest(), - BYORevisionPostUpgradeTest(), - CreateNewServicePostUpgradeTest(), - InitialScalePostUpgradeTest(), - }, - PostDowngrade: []pkgupgrade.Operation{ - ServicePostDowngradeTest(), - CreateNewServicePostDowngradeTest(), - }, - Continual: []pkgupgrade.BackgroundOperation{ - ProbeTest(), - AutoscaleSustainingTest(c.Log), - AutoscaleSustainingWithTBCTest(c.Log), - }, + PreUpgrade: ServingPreUpgradeTests(), + PostUpgrade: ServingPostUpgradeTests(), + PostDowngrade: ServingPostDowngradeTests(), + Continual: ContinualTests(), }, Installations: pkgupgrade.Installations{ Base: []pkgupgrade.Operation{ From b08399a81b93cbb4c61c88b7a9747e89c7c73bff Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 1 Dec 2020 15:19:05 +0100 Subject: [PATCH 03/18] Define curPods and targetPods constants in upgrade tests --- test/upgrade/autoscaler.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go index cdc4c58de79e..b031156f4347 100644 --- a/test/upgrade/autoscaler.go +++ b/test/upgrade/autoscaler.go @@ -32,6 +32,8 @@ import ( const ( containerConcurrency = 6 targetUtilization = 0.7 + curPods = 1 + targetPods = 10 ) // AutoscaleSustainingTest checks that when traffic increases a knative app @@ -49,7 +51,7 @@ func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. })) grp.Go(func() error { - return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, 1, 10, stopCh, false /* quick */) + return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) }) }, func(c pkgupgrade.Context) { @@ -78,7 +80,7 @@ func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. })) grp.Go(func() error { - return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, 1, 10, stopCh, false /* quick */) + return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) }) }, func(c pkgupgrade.Context) { From 27f5b043183bb2bc9f97e7a1146448db2ab540de Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 1 Dec 2020 15:23:01 +0100 Subject: [PATCH 04/18] Fix imports and licences --- test/upgrade/continual.go | 16 ++++++++++++++++ test/upgrade/postdowngrade.go | 2 +- test/upgrade/postupgrade.go | 2 +- test/upgrade/preupgrade.go | 5 +++-- test/upgrade/upgrade.go | 2 +- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/test/upgrade/continual.go b/test/upgrade/continual.go index 397503f0b21b..82359d0231a3 100644 --- a/test/upgrade/continual.go +++ b/test/upgrade/continual.go @@ -1,3 +1,19 @@ +/* +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package upgrade import pkgupgrade "knative.dev/pkg/test/upgrade" diff --git a/test/upgrade/postdowngrade.go b/test/upgrade/postdowngrade.go index ec2a08179549..6d51b3f64788 100644 --- a/test/upgrade/postdowngrade.go +++ b/test/upgrade/postdowngrade.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/test/upgrade/postupgrade.go b/test/upgrade/postupgrade.go index a7a073c42cc1..d5ca6a44adfe 100644 --- a/test/upgrade/postupgrade.go +++ b/test/upgrade/postupgrade.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/test/upgrade/preupgrade.go b/test/upgrade/preupgrade.go index c0f635439942..6a20fc855fbb 100644 --- a/test/upgrade/preupgrade.go +++ b/test/upgrade/preupgrade.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,6 +17,8 @@ limitations under the License. package upgrade import ( + "testing" + pkgupgrade "knative.dev/pkg/test/upgrade" "knative.dev/serving/pkg/apis/autoscaling" revisionresourcenames "knative.dev/serving/pkg/reconciler/revision/resources/names" @@ -24,7 +26,6 @@ import ( "knative.dev/serving/test" "knative.dev/serving/test/e2e" v1test "knative.dev/serving/test/v1" - "testing" ) func ServingPreUpgradeTests() []pkgupgrade.Operation { diff --git a/test/upgrade/upgrade.go b/test/upgrade/upgrade.go index e91cfa02358f..a5fcfaa98e41 100644 --- a/test/upgrade/upgrade.go +++ b/test/upgrade/upgrade.go @@ -1,5 +1,5 @@ /* -Copyright 2018 The Knative Authors +Copyright 2020 The Knative Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 63a05db40624907f486a56d5879dac121beda3ea Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 1 Dec 2020 15:27:22 +0100 Subject: [PATCH 05/18] Fix lint - comments on new exported functions --- test/upgrade/continual.go | 1 + test/upgrade/postdowngrade.go | 1 + test/upgrade/postupgrade.go | 1 + test/upgrade/preupgrade.go | 1 + 4 files changed, 4 insertions(+) diff --git a/test/upgrade/continual.go b/test/upgrade/continual.go index 82359d0231a3..2df35d25ebd9 100644 --- a/test/upgrade/continual.go +++ b/test/upgrade/continual.go @@ -18,6 +18,7 @@ package upgrade import pkgupgrade "knative.dev/pkg/test/upgrade" +// ContinualTests is an umbrella function for grouping all Serving continual/background tests. func ContinualTests() []pkgupgrade.BackgroundOperation { return []pkgupgrade.BackgroundOperation{ ProbeTest(), diff --git a/test/upgrade/postdowngrade.go b/test/upgrade/postdowngrade.go index 6d51b3f64788..61a8aee8e81f 100644 --- a/test/upgrade/postdowngrade.go +++ b/test/upgrade/postdowngrade.go @@ -31,6 +31,7 @@ import ( v1test "knative.dev/serving/test/v1" ) +// ServingPostDowngradeTests is an umbrella function for grouping all Serving post-downgrade tests. func ServingPostDowngradeTests() []pkgupgrade.Operation { return []pkgupgrade.Operation{ ServicePostDowngradeTest(), diff --git a/test/upgrade/postupgrade.go b/test/upgrade/postupgrade.go index d5ca6a44adfe..711f2fd1dc22 100644 --- a/test/upgrade/postupgrade.go +++ b/test/upgrade/postupgrade.go @@ -32,6 +32,7 @@ import ( v1test "knative.dev/serving/test/v1" ) +// ServingPostUpgradeTests is an umbrella function for grouping all Serving post-upgrade tests. func ServingPostUpgradeTests() []pkgupgrade.Operation { return []pkgupgrade.Operation{ ServicePostUpgradeTest(), diff --git a/test/upgrade/preupgrade.go b/test/upgrade/preupgrade.go index 6a20fc855fbb..77fb99dfbb1f 100644 --- a/test/upgrade/preupgrade.go +++ b/test/upgrade/preupgrade.go @@ -28,6 +28,7 @@ import ( v1test "knative.dev/serving/test/v1" ) +// ServingPreUpgradeTests is an umbrella function for grouping all Serving pre-upgrade tests. func ServingPreUpgradeTests() []pkgupgrade.Operation { return []pkgupgrade.Operation{ ServicePreUpgradeTest(), From 272a4205f0d9a2acdf40fe788a5f28e4624bd9c7 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Wed, 2 Dec 2020 08:46:36 +0100 Subject: [PATCH 06/18] Split AssertAutoscaleUpToNumPods into setup and wait parts --- test/e2e/autoscale.go | 36 +++++++++++++++++++----------------- test/e2e/autoscale_test.go | 29 +++++++---------------------- test/e2e/grpc_test.go | 2 +- test/ha/autoscaler_test.go | 8 ++------ test/upgrade/autoscaler.go | 8 ++------ 5 files changed, 31 insertions(+), 52 deletions(-) diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index a8efb6bdf3c3..748e49bb6cb9 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -260,14 +260,14 @@ func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization } } -func assertScaleDown(ctx *TestContext, logf logging.FormatLogger) error { +func assertScaleDown(ctx *TestContext) { deploymentName := resourcenames.Deployment(ctx.resources.Revision) if err := WaitForScaleToZero(ctx.t, deploymentName, ctx.clients); err != nil { - return fmt.Errorf("unable to observe the Deployment named %s scaling down: %w", deploymentName, err) + ctx.t.Fatalf("Unable to observe the Deployment named %s scaling down: %v", deploymentName, err) } // Account for the case where scaling up uses all available pods. - logf("Wait for all pods to terminate.") + ctx.t.Logf("Wait for all pods to terminate.") if err := pkgTest.WaitForPodListState( context.Background(), @@ -283,17 +283,15 @@ func assertScaleDown(ctx *TestContext, logf logging.FormatLogger) error { return true, nil }, "WaitForAvailablePods", test.ServingNamespace); err != nil { - return fmt.Errorf("waiting for Pod.List to have no non-Evicted pods of %q: %w", deploymentName, err) + ctx.t.Fatalf("Waiting for Pod.List to have no non-Evicted pods of %q: %v", deploymentName, err) } - logf("The Revision should remain ready after scaling to zero.") + ctx.t.Logf("The Revision should remain ready after scaling to zero.") if err := v1test.CheckRevisionState(ctx.clients.ServingClient, ctx.names.Revision, v1test.IsRevisionReady); err != nil { - return fmt.Errorf("the Revision %s did not stay Ready after scaling down to zero: %w", ctx.names.Revision, err) + ctx.t.Fatalf("The Revision %s did not stay Ready after scaling down to zero: %v", ctx.names.Revision, err) } - logf("Scaled down.") - - return nil + ctx.t.Logf("Scaled down.") } func numberOfReadyPods(ctx *TestContext, logf logging.FormatLogger) (float64, error) { @@ -373,14 +371,24 @@ func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minP // sustains there until the `done` channel sends a signal. // The given `duration` is how long the traffic will be generated. You must make sure that the signal // from the given `done` channel will be sent within the `duration`. -func AssertAutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, targetPods float64, done <-chan time.Time, quick bool) error { +func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) { + var grp errgroup.Group + AutoscaleUpToNumPods(ctx, ctx.t.Logf, curPods, targetPods, grp, done, quick) + if err := grp.Wait(); err != nil { + ctx.t.Fatal(err) + } +} + +// AutoscaleUpToNumPods starts the traffic for AssertAutoscaleUpToNumPods and returns +// an error group to wait for. Starting the routines is separated from waiting for +// easy re-use in other places (e.g. upgrade tests). +func AutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, targetPods float64, grp errgroup.Group, done <-chan time.Time, quick bool) { // Relax the bounds to reduce the flakiness caused by sampling in the autoscaling algorithm. // Also adjust the values by the target utilization values. minPods := math.Floor(curPods/ctx.targetUtilization) - 1 maxPods := math.Ceil(math.Ceil(targetPods/ctx.targetUtilization) * 1.1) stopChan := make(chan struct{}) - var grp errgroup.Group grp.Go(func() error { switch ctx.metric { case autoscaling.RPS: @@ -394,10 +402,4 @@ func AssertAutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, cur defer close(stopChan) return checkPodScale(ctx, logf, targetPods, minPods, maxPods, done, quick) }) - - if err := grp.Wait(); err != nil { - return err - } - - return nil } diff --git a/test/e2e/autoscale_test.go b/test/e2e/autoscale_test.go index d967b9bfd369..e8100877f17e 100644 --- a/test/e2e/autoscale_test.go +++ b/test/e2e/autoscale_test.go @@ -43,17 +43,10 @@ func TestAutoscaleUpDownUp(t *testing.T) { ctx := SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization) test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) - if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 2, time.After(60*time.Second), true /* quick */); err != nil { - t.Fatal(err) - } - if err := assertScaleDown(ctx, t.Logf); err != nil { - t.Fatal(err) - } - if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 0, 2, time.After(60*time.Second), true /* quick */); err != nil { - t.Fatal(err) - } + AssertAutoscaleUpToNumPods(ctx, 1, 2, time.After(60*time.Second), true /* quick */) + assertScaleDown(ctx) + AssertAutoscaleUpToNumPods(ctx, 0, 2, time.After(60*time.Second), true /* quick */) } - func TestAutoscaleUpCountPods(t *testing.T) { t.Parallel() runAutoscaleUpCountPods(t, autoscaling.KPA, autoscaling.Concurrency) @@ -83,17 +76,11 @@ func runAutoscaleUpCountPods(t *testing.T, class, metric string) { // boskos cluster to propagate the state. See #10218. // Assert the number of expected replicas is between n-1 and n+1, where n is the # of desired replicas for 60s. // Assert the number of expected replicas is n and n+1 at the end of 90s, where n is the # of desired replicas. - if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 2, time.After(90*time.Second), true /* quick */); err != nil { - t.Fatal(err) - } + AssertAutoscaleUpToNumPods(ctx, 1, 2, time.After(90*time.Second), true /* quick */) // Increase workload scale to 3 replicas, assert between [n-1, n+1] during scale up, assert between [n, n+1] after scaleup. - if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 2, 3, time.After(90*time.Second), true /* quick */); err != nil { - t.Fatal(err) - } + AssertAutoscaleUpToNumPods(ctx, 2, 3, time.After(90*time.Second), true /* quick */) // Increase workload scale to 4 replicas, assert between [n-1, n+1] during scale up, assert between [n, n+1] after scaleup. - if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 3, 4, time.After(90*time.Second), true /* quick */); err != nil { - t.Fatal(err) - } + AssertAutoscaleUpToNumPods(ctx, 3, 4, time.After(90*time.Second), true /* quick */) } func TestAutoscaleSustaining(t *testing.T) { @@ -105,9 +92,7 @@ func TestAutoscaleSustaining(t *testing.T) { ctx := SetupSvc(t, autoscaling.KPA, autoscaling.Concurrency, containerConcurrency, targetUtilization) test.EnsureTearDown(t, ctx.Clients(), ctx.Names()) - if err := AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 10, time.After(2*time.Minute), false /* quick */); err != nil { - t.Fatal(err) - } + AssertAutoscaleUpToNumPods(ctx, 1, 10, time.After(2*time.Minute), false /* quick */) } func TestTargetBurstCapacity(t *testing.T) { diff --git a/test/e2e/grpc_test.go b/test/e2e/grpc_test.go index 29b3917d7768..959d86291528 100644 --- a/test/e2e/grpc_test.go +++ b/test/e2e/grpc_test.go @@ -111,7 +111,7 @@ func autoscaleTest(ctx *TestContext, host, domain string) { ctx.targetUtilization = targetUtilization assertGRPCAutoscaleUpToNumPods(ctx, 1, 2, 60*time.Second, host, domain) - assertScaleDown(ctx, ctx.t.Logf) + assertScaleDown(ctx) assertGRPCAutoscaleUpToNumPods(ctx, 0, 2, 60*time.Second, host, domain) } diff --git a/test/ha/autoscaler_test.go b/test/ha/autoscaler_test.go index 0d8e108daf39..f548ef0bb0ec 100644 --- a/test/ha/autoscaler_test.go +++ b/test/ha/autoscaler_test.go @@ -90,9 +90,7 @@ func TestAutoscalerHA(t *testing.T) { } t.Log("Verifying the old revision can still be scaled from 0 to some number after leader change") - if err := e2e.AssertAutoscaleUpToNumPods(ctx, t.Logf, 0, 3, time.After(60*time.Second), true /* quick */); err != nil { - t.Fatal(err) - } + e2e.AssertAutoscaleUpToNumPods(ctx, 0, 3, time.After(60*time.Second), true /* quick */) t.Log("Updating the Service after selecting new leader controller in order to generate a new revision") names.Image = test.PizzaPlanet2 @@ -111,7 +109,5 @@ func TestAutoscalerHA(t *testing.T) { t.Log("Verifying the new revision can be scaled up") ctx.SetNames(names) ctx.SetResources(resources) - if err := e2e.AssertAutoscaleUpToNumPods(ctx, t.Logf, 1, 3, time.After(60*time.Second), true /* quick */); err != nil { - t.Fatal(err) - } + e2e.AssertAutoscaleUpToNumPods(ctx, 1, 3, time.After(60*time.Second), true /* quick */) } diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go index b031156f4347..002ca6cac135 100644 --- a/test/upgrade/autoscaler.go +++ b/test/upgrade/autoscaler.go @@ -50,9 +50,7 @@ func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. })) - grp.Go(func() error { - return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) - }) + e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, grp, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) @@ -79,9 +77,7 @@ func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. })) - grp.Go(func() error { - return e2e.AssertAutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) - }) + e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, grp, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) From 9ef05fcaf83d1c43109cb96560f1243189a60672 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Wed, 2 Dec 2020 10:01:01 +0100 Subject: [PATCH 07/18] Pass errgroup by reference --- test/e2e/autoscale.go | 8 +++++--- test/upgrade/autoscaler.go | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index 748e49bb6cb9..f5898f71d00e 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -372,8 +372,7 @@ func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minP // The given `duration` is how long the traffic will be generated. You must make sure that the signal // from the given `done` channel will be sent within the `duration`. func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) { - var grp errgroup.Group - AutoscaleUpToNumPods(ctx, ctx.t.Logf, curPods, targetPods, grp, done, quick) + grp := AutoscaleUpToNumPods(ctx, ctx.t.Logf, curPods, targetPods, done, quick) if err := grp.Wait(); err != nil { ctx.t.Fatal(err) } @@ -382,13 +381,14 @@ func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, d // AutoscaleUpToNumPods starts the traffic for AssertAutoscaleUpToNumPods and returns // an error group to wait for. Starting the routines is separated from waiting for // easy re-use in other places (e.g. upgrade tests). -func AutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, targetPods float64, grp errgroup.Group, done <-chan time.Time, quick bool) { +func AutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, targetPods float64, done <-chan time.Time, quick bool) *errgroup.Group { // Relax the bounds to reduce the flakiness caused by sampling in the autoscaling algorithm. // Also adjust the values by the target utilization values. minPods := math.Floor(curPods/ctx.targetUtilization) - 1 maxPods := math.Ceil(math.Ceil(targetPods/ctx.targetUtilization) * 1.1) stopChan := make(chan struct{}) + grp := &errgroup.Group{} grp.Go(func() error { switch ctx.metric { case autoscaling.RPS: @@ -402,4 +402,6 @@ func AutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, defer close(stopChan) return checkPodScale(ctx, logf, targetPods, minPods, maxPods, done, quick) }) + + return grp } diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go index 002ca6cac135..15e91cdcecde 100644 --- a/test/upgrade/autoscaler.go +++ b/test/upgrade/autoscaler.go @@ -41,7 +41,7 @@ const ( // it is switching modes between normal and panic. func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { var ctx *e2e.TestContext - var grp errgroup.Group + var grp *errgroup.Group stopCh := make(chan time.Time) return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingTest", func(c pkgupgrade.Context) { @@ -50,7 +50,7 @@ func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. })) - e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, grp, stopCh, false /* quick */) + grp = e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) @@ -68,7 +68,7 @@ func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { // in the path a knative app scales up and sustains the scale. func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { var ctx *e2e.TestContext - var grp errgroup.Group + var grp *errgroup.Group stopCh := make(chan time.Time) return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingWithTBCTest", func(c pkgupgrade.Context) { @@ -77,7 +77,7 @@ func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. })) - e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, grp, stopCh, false /* quick */) + grp = e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) From 150bac8ed063bd54d072320abfe82825c87dd380 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Wed, 2 Dec 2020 10:36:23 +0100 Subject: [PATCH 08/18] Fix lint error --- vendor/knative.dev/hack/shell/project.go | 32 ++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/vendor/knative.dev/hack/shell/project.go b/vendor/knative.dev/hack/shell/project.go index 2d7ba9738b08..f4cd1681adc0 100644 --- a/vendor/knative.dev/hack/shell/project.go +++ b/vendor/knative.dev/hack/shell/project.go @@ -18,12 +18,21 @@ package shell import ( "errors" + "fmt" "path" + "regexp" "runtime" ) -// ErrCantGetCaller is raised when we can't calculate a caller of NewProjectLocation. -var ErrCantGetCaller = errors.New("can't get caller") +var ( + // ErrCantGetCaller is raised when we can't calculate a caller of NewProjectLocation. + ErrCantGetCaller = errors.New("can't get caller") + + // ErrCallerNotAllowed is raised when user tries to use this shell-out package + // outside of allowed places. This package is deprecated from start and was + // introduced to allow rewriting of shell code to Golang in small chunks. + ErrCallerNotAllowed = errors.New("don't try use knative.dev/hack/shell package outside of allowed places") +) // NewProjectLocation creates a ProjectLocation that is used to calculate // relative paths within the project. @@ -32,6 +41,10 @@ func NewProjectLocation(pathToRoot string) (ProjectLocation, error) { if !ok { return nil, ErrCantGetCaller } + err := ensureIsValid(filename) + if err != nil { + return nil, err + } return &callerLocation{ caller: filename, pathToRoot: pathToRoot, @@ -50,3 +63,18 @@ type callerLocation struct { caller string pathToRoot string } + +func ensureIsValid(filename string) error { + validPaths := []string{ + "knative.+/test/upgrade/", + "knative(:?\\.dev/|-)hack/shell/", + } + for _, validPath := range validPaths { + r := regexp.MustCompile(validPath) + if loc := r.FindStringIndex(filename); loc != nil { + return nil + } + } + return fmt.Errorf("%w, tried using from: %s", + ErrCallerNotAllowed, filename) +} From 75acab530d80ca24b0ac48a01fe1a40e0fedd0c1 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Thu, 3 Dec 2020 14:08:03 +0100 Subject: [PATCH 09/18] Move logf into TestContext --- test/e2e/autoscale.go | 56 ++++++++++++++++------------- test/e2e/grpc_test.go | 1 + test/e2e/service_to_service_test.go | 1 + test/e2e/websocket_test.go | 1 + test/upgrade/autoscaler.go | 6 ++-- 5 files changed, 38 insertions(+), 27 deletions(-) diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index f5898f71d00e..7b40a26a24c8 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -62,6 +62,7 @@ const ( // TestContext includes context for autoscaler testing. type TestContext struct { t *testing.T + logf logging.FormatLogger clients *test.Clients names *test.ResourceNames resources *v1test.ResourceObjects @@ -95,6 +96,11 @@ func (ctx *TestContext) SetNames(names *test.ResourceNames) { ctx.names = names } +// SetLogger sets the logger of the TestContext. +func (ctx *TestContext) SetLogger(logf logging.FormatLogger) { + ctx.logf = logf +} + func getVegetaTarget(kubeClientset kubernetes.Interface, domain, endpointOverride string, resolvable bool) (vegeta.Target, error) { if resolvable { return vegeta.Target{ @@ -122,7 +128,6 @@ func getVegetaTarget(kubeClientset kubernetes.Interface, domain, endpointOverrid func generateTraffic( ctx *TestContext, - logf logging.FormatLogger, attacker *vegeta.Attacker, pacer vegeta.Pacer, stopChan chan struct{}) error { @@ -144,7 +149,7 @@ func generateTraffic( for { select { case <-stopChan: - logf("Stopping generateTraffic") + ctx.logf("Stopping generateTraffic") successRate := float64(1) if totalRequests > 0 { successRate = float64(successfulRequests) / float64(totalRequests) @@ -156,14 +161,14 @@ func generateTraffic( return nil case res, ok := <-results: if !ok { - logf("Time is up; done") + ctx.logf("Time is up; done") return nil } totalRequests++ if res.Code != http.StatusOK { - logf("Status = %d, want: 200", res.Code) - logf("URL: %s Duration: %v Error: %s Body:\n%s", res.URL, res.Latency, res.Error, string(res.Body)) + ctx.logf("Status = %d, want: 200", res.Code) + ctx.logf("URL: %s Duration: %v Error: %s Body:\n%s", res.URL, res.Latency, res.Error, string(res.Body)) continue } successfulRequests++ @@ -171,23 +176,23 @@ func generateTraffic( } } -func generateTrafficAtFixedConcurrency(ctx *TestContext, logf logging.FormatLogger, concurrency int, stopChan chan struct{}) error { +func generateTrafficAtFixedConcurrency(ctx *TestContext, concurrency int, stopChan chan struct{}) error { pacer := vegeta.ConstantPacer{} // Sends requests as quickly as possible, capped by MaxWorkers below. attacker := vegeta.NewAttacker( vegeta.Timeout(0), // No timeout is enforced at all. vegeta.Workers(uint64(concurrency)), vegeta.MaxWorkers(uint64(concurrency))) - logf("Maintaining %d concurrent requests.", concurrency) - return generateTraffic(ctx, logf, attacker, pacer, stopChan) + ctx.logf("Maintaining %d concurrent requests.", concurrency) + return generateTraffic(ctx, attacker, pacer, stopChan) } -func generateTrafficAtFixedRPS(ctx *TestContext, logf logging.FormatLogger, rps int, stopChan chan struct{}) error { +func generateTrafficAtFixedRPS(ctx *TestContext, rps int, stopChan chan struct{}) error { pacer := vegeta.ConstantPacer{Freq: rps, Per: time.Second} attacker := vegeta.NewAttacker(vegeta.Timeout(0)) // No timeout is enforced at all. - logf("Maintaining %v RPS.", rps) - return generateTraffic(ctx, logf, attacker, pacer, stopChan) + ctx.logf("Maintaining %v RPS.", rps) + return generateTraffic(ctx, attacker, pacer, stopChan) } func toPercentageString(f float64) string { @@ -251,6 +256,7 @@ func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization return &TestContext{ t: t, + logf: t.Logf, clients: clients, names: names, resources: resources, @@ -294,16 +300,16 @@ func assertScaleDown(ctx *TestContext) { ctx.t.Logf("Scaled down.") } -func numberOfReadyPods(ctx *TestContext, logf logging.FormatLogger) (float64, error) { +func numberOfReadyPods(ctx *TestContext) (float64, error) { // SKS name matches that of revision. n := ctx.resources.Revision.Name sks, err := ctx.clients.NetworkingClient.ServerlessServices.Get(context.Background(), n, metav1.GetOptions{}) if err != nil { - logf("Error getting SKS %q: %v", n, err) + ctx.logf("Error getting SKS %q: %v", n, err) return 0, fmt.Errorf("error retrieving sks %q: %w", n, err) } if sks.Status.PrivateServiceName == "" { - logf("SKS %s has not yet reconciled", n) + ctx.logf("SKS %s has not yet reconciled", n) // Not an error, but no pods either. return 0, nil } @@ -315,7 +321,7 @@ func numberOfReadyPods(ctx *TestContext, logf logging.FormatLogger) (float64, er return float64(resources.ReadyAddressCount(eps)), nil } -func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minPods, maxPods float64, done <-chan time.Time, quick bool) error { +func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done <-chan time.Time, quick bool) error { // Short-circuit traffic generation once we exit from the check logic. ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() @@ -325,12 +331,12 @@ func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minP case <-ticker.C: // Each 2 second, check that the number of pods is at least `minPods`. `minPods` is increasing // to verify that the number of pods doesn't go down while we are scaling up. - got, err := numberOfReadyPods(ctx, logf) + got, err := numberOfReadyPods(ctx) if err != nil { return err } mes := fmt.Sprintf("revision %q #replicas: %v, want at least: %v", ctx.resources.Revision.Name, got, minPods) - logf(mes) + ctx.logf(mes) // verify that the number of pods doesn't go down while we are scaling up. if got < minPods { return errors.New("interim scale didn't fulfill constraints: " + mes) @@ -338,7 +344,7 @@ func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minP // A quick test succeeds when the number of pods scales up to `targetPods` // (and, as an extra check, no more than `maxPods`). if quick && got >= targetPods && got <= maxPods { - logf("Quick Mode: got %v >= %v", got, targetPods) + ctx.logf("Quick Mode: got %v >= %v", got, targetPods) return nil } if minPods < targetPods-1 { @@ -349,13 +355,13 @@ func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minP case <-done: // The test duration is over. Do a last check to verify that the number of pods is at `targetPods` // (with a little room for de-flakiness). - got, err := numberOfReadyPods(ctx, logf) + got, err := numberOfReadyPods(ctx) if err != nil { return fmt.Errorf("failed to fetch number of ready pods: %w", err) } mes := fmt.Sprintf("got %v replicas, expected between [%v, %v] replicas for revision %s", got, targetPods-1, maxPods, ctx.resources.Revision.Name) - logf(mes) + ctx.logf(mes) if got < targetPods-1 || got > maxPods { return errors.New("final scale didn't fulfill constraints: " + mes) } @@ -372,7 +378,7 @@ func checkPodScale(ctx *TestContext, logf logging.FormatLogger, targetPods, minP // The given `duration` is how long the traffic will be generated. You must make sure that the signal // from the given `done` channel will be sent within the `duration`. func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) { - grp := AutoscaleUpToNumPods(ctx, ctx.t.Logf, curPods, targetPods, done, quick) + grp := AutoscaleUpToNumPods(ctx, curPods, targetPods, done, quick) if err := grp.Wait(); err != nil { ctx.t.Fatal(err) } @@ -381,7 +387,7 @@ func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, d // AutoscaleUpToNumPods starts the traffic for AssertAutoscaleUpToNumPods and returns // an error group to wait for. Starting the routines is separated from waiting for // easy re-use in other places (e.g. upgrade tests). -func AutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, targetPods float64, done <-chan time.Time, quick bool) *errgroup.Group { +func AutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) *errgroup.Group { // Relax the bounds to reduce the flakiness caused by sampling in the autoscaling algorithm. // Also adjust the values by the target utilization values. minPods := math.Floor(curPods/ctx.targetUtilization) - 1 @@ -392,15 +398,15 @@ func AutoscaleUpToNumPods(ctx *TestContext, logf logging.FormatLogger, curPods, grp.Go(func() error { switch ctx.metric { case autoscaling.RPS: - return generateTrafficAtFixedRPS(ctx, logf, int(targetPods*float64(ctx.targetValue)), stopChan) + return generateTrafficAtFixedRPS(ctx, int(targetPods*float64(ctx.targetValue)), stopChan) default: - return generateTrafficAtFixedConcurrency(ctx, logf, int(targetPods*float64(ctx.targetValue)), stopChan) + return generateTrafficAtFixedConcurrency(ctx, int(targetPods*float64(ctx.targetValue)), stopChan) } }) grp.Go(func() error { defer close(stopChan) - return checkPodScale(ctx, logf, targetPods, minPods, maxPods, done, quick) + return checkPodScale(ctx, targetPods, minPods, maxPods, done, quick) }) return grp diff --git a/test/e2e/grpc_test.go b/test/e2e/grpc_test.go index 959d86291528..289ff962eb9d 100644 --- a/test/e2e/grpc_test.go +++ b/test/e2e/grpc_test.go @@ -362,6 +362,7 @@ func testGRPC(t *testing.T, f grpcTest, fopts ...rtesting.ServiceOption) { f(&TestContext{ t: t, + logf: t.Logf, clients: clients, names: names, resources: resources, diff --git a/test/e2e/service_to_service_test.go b/test/e2e/service_to_service_test.go index 4e9bf49a0735..593f769eed02 100644 --- a/test/e2e/service_to_service_test.go +++ b/test/e2e/service_to_service_test.go @@ -243,6 +243,7 @@ func testSvcToSvcCallViaActivator(t *testing.T, clients *test.Clients, injectA b // Wait for the activator endpoints to equalize. if err := waitForActivatorEndpoints(&TestContext{ t: t, + logf: t.Logf, resources: resources, clients: clients, }); err != nil { diff --git a/test/e2e/websocket_test.go b/test/e2e/websocket_test.go index cf10defe0520..fa108efcf8d3 100644 --- a/test/e2e/websocket_test.go +++ b/test/e2e/websocket_test.go @@ -170,6 +170,7 @@ func TestWebSocketViaActivator(t *testing.T) { // Wait for the activator endpoints to equalize. if err := waitForActivatorEndpoints(&TestContext{ t: t, + logf: t.Logf, resources: resources, clients: clients, }); err != nil { diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go index 15e91cdcecde..86eaae22646b 100644 --- a/test/upgrade/autoscaler.go +++ b/test/upgrade/autoscaler.go @@ -50,7 +50,8 @@ func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. })) - grp = e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) + ctx.SetLogger(c.Log.Infof) + grp = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) @@ -77,7 +78,8 @@ func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { rtesting.WithConfigAnnotations(map[string]string{ autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. })) - grp = e2e.AutoscaleUpToNumPods(ctx, c.Log.Infof, curPods, targetPods, stopCh, false /* quick */) + ctx.SetLogger(c.Log.Infof) + grp = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) From d0d348a2e23501cf0e9808b30d59208fa7cbd951 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Thu, 3 Dec 2020 21:46:17 +0100 Subject: [PATCH 10/18] Remaining fixes for moving logf to TestContext --- test/e2e/autoscale_test.go | 6 +++--- test/e2e/grpc_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/autoscale_test.go b/test/e2e/autoscale_test.go index e8100877f17e..0ee16e114e8e 100644 --- a/test/e2e/autoscale_test.go +++ b/test/e2e/autoscale_test.go @@ -122,7 +122,7 @@ func TestTargetBurstCapacity(t *testing.T) { defer close(stopCh) grp.Go(func() error { - return generateTrafficAtFixedConcurrency(ctx, t.Logf, 7, stopCh) + return generateTrafficAtFixedConcurrency(ctx, 7, stopCh) }) // Wait for the activator endpoints to equalize. @@ -132,13 +132,13 @@ func TestTargetBurstCapacity(t *testing.T) { // Start second load generator. grp.Go(func() error { - return generateTrafficAtFixedConcurrency(ctx, t.Logf, 5, stopCh) + return generateTrafficAtFixedConcurrency(ctx, 5, stopCh) }) // Wait for two stable pods. obsScale := 0.0 if err := wait.Poll(250*time.Millisecond, 2*cfg.StableWindow, func() (bool, error) { - obsScale, err = numberOfReadyPods(ctx, t.Logf) + obsScale, err = numberOfReadyPods(ctx) if err != nil { return false, err } diff --git a/test/e2e/grpc_test.go b/test/e2e/grpc_test.go index 289ff962eb9d..b16242ac7944 100644 --- a/test/e2e/grpc_test.go +++ b/test/e2e/grpc_test.go @@ -256,7 +256,7 @@ func assertGRPCAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float6 grp.Go(func() error { defer close(stopChan) - return checkPodScale(ctx, ctx.t.Logf, targetPods, minPods, maxPods, time.After(duration), true /* quick */) + return checkPodScale(ctx, targetPods, minPods, maxPods, time.After(duration), true /* quick */) }) if err := grp.Wait(); err != nil { From 7dc161c18301b03c8f6a8e29ac549d3efcfda428 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Fri, 4 Dec 2020 13:12:58 +0100 Subject: [PATCH 11/18] Call logf on context directly --- test/e2e/autoscale.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index 7b40a26a24c8..12e4adf55ad5 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -273,7 +273,7 @@ func assertScaleDown(ctx *TestContext) { } // Account for the case where scaling up uses all available pods. - ctx.t.Logf("Wait for all pods to terminate.") + ctx.logf("Wait for all pods to terminate.") if err := pkgTest.WaitForPodListState( context.Background(), @@ -292,12 +292,12 @@ func assertScaleDown(ctx *TestContext) { ctx.t.Fatalf("Waiting for Pod.List to have no non-Evicted pods of %q: %v", deploymentName, err) } - ctx.t.Logf("The Revision should remain ready after scaling to zero.") + ctx.logf("The Revision should remain ready after scaling to zero.") if err := v1test.CheckRevisionState(ctx.clients.ServingClient, ctx.names.Revision, v1test.IsRevisionReady); err != nil { ctx.t.Fatalf("The Revision %s did not stay Ready after scaling down to zero: %v", ctx.names.Revision, err) } - ctx.t.Logf("Scaled down.") + ctx.logf("Scaled down.") } func numberOfReadyPods(ctx *TestContext) (float64, error) { From 35ce9f4a1f6ee64501ac7ff678e7969f2871dc07 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Fri, 4 Dec 2020 13:31:43 +0100 Subject: [PATCH 12/18] Update readme --- test/upgrade/README.md | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/test/upgrade/README.md b/test/upgrade/README.md index 604f56c7f81f..d4d06302a414 100644 --- a/test/upgrade/README.md +++ b/test/upgrade/README.md @@ -25,40 +25,41 @@ At a high level, we want to do this: 1. Install knative at HEAD. 1. Test those resources, verify that we didn’t break anything. -To achieve that, we just have three separate build tags: +To achieve that, we utilize the [upgrade test framework](https://github.com/knative/pkg/tree/master/test/upgrade). +The framework runs tests in the following phases: 1. Install the latest release from GitHub. -1. Run the `preupgrade` tests in this directory. -1. Install at HEAD (`ko apply -Rf config/core/`). -1. Run the `postupgrade` tests in this directory. +1. Run the `PreUpgrade` tests. +1. Start the `Continual` tests. +1. Install from the HEAD of the master branch. +1. Run the `PostUpgrade` tests. 1. Install the latest release from GitHub. -1. Run the `postdowngrade` tests in this directory. +1. Run the `PostDowngrade` tests. +1. End the `Continual` tests and collect results. ## Tests -### Service test +#### PreUpgrade -This was stolen from the conformance tests but stripped down to check fewer -things. +Create a Service pointing to `image1`, ensure it responds correctly. -#### preupgrade - -Create a RunLatest Service pointing to `image1`, ensure it responds correctly. - -#### postupgrade +#### PostUpgrade Ensure the Service still responds correctly after upgrading. Update it to point -to `image2`, ensure it responds correctly. +to `image2`, ensure it responds correctly. Ensure that a new service +can be created after downgrade. -#### postdowngrade +#### PostDowngrade Ensure the Service still responds correctly after downgrading. Update it to -point back to `image1`, ensure it responds correctly. +point back to `image1`, ensure it responds correctly. Ensure that a new service +can be created after downgrade. -### Probe test +### Continual In order to verify that we don't have data-plane unavailability during our control-plane outages (when we're upgrading the knative/serving installation), we run a prober test that continually sends requests to a service during the entire upgrade process. When the upgrade completes, we make sure that none of -those requests failed. +those requests failed. We also run autoscale tests to ensure that autoscaling +works correctly during upgrades/downgrades. From 7b6023159279485cc8782aa2ea9ebd70a2319dd6 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Fri, 4 Dec 2020 15:04:28 +0100 Subject: [PATCH 13/18] Remove the comment and fix imports --- test/e2e/autoscale.go | 2 -- test/upgrade/autoscaler.go | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index 12e4adf55ad5..4d9a5fa83a38 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -202,8 +202,6 @@ func toPercentageString(f float64) string { // SetupSvc creates a new service, with given service options. // It returns a TestContext that has resources, K8s clients and other needed // data points. -// It sets up EnsureTearDown to ensure that resources are cleaned up when the -// test terminates. func SetupSvc(t *testing.T, class, metric string, target int, targetUtilization float64, fopts ...rtesting.ServiceOption) *TestContext { t.Helper() clients := Setup(t) diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go index 86eaae22646b..39e32f97f180 100644 --- a/test/upgrade/autoscaler.go +++ b/test/upgrade/autoscaler.go @@ -20,12 +20,10 @@ import ( "time" "golang.org/x/sync/errgroup" - "knative.dev/serving/test" - pkgupgrade "knative.dev/pkg/test/upgrade" - "knative.dev/serving/pkg/apis/autoscaling" rtesting "knative.dev/serving/pkg/testing/v1" + "knative.dev/serving/test" "knative.dev/serving/test/e2e" ) From 6029f1c6b8f81a07e03e1b8e59dd8f9c8da45c83 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 8 Dec 2020 10:59:55 +0100 Subject: [PATCH 14/18] Update modules after running update-codegen --- vendor/modules.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/vendor/modules.txt b/vendor/modules.txt index eb47c42ae49e..5ac625083816 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -987,6 +987,7 @@ knative.dev/caching/pkg/client/listers/caching/v1alpha1 # knative.dev/hack v0.0.0-20201214230143-4ed1ecb8db24 ## explicit knative.dev/hack +knative.dev/hack/shell # knative.dev/networking v0.0.0-20201223042504-b9e08949dfbc ## explicit knative.dev/networking/config From 0449e67514471ac352780a626b93cbc7bbebaa9c Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Wed, 9 Dec 2020 14:54:51 +0100 Subject: [PATCH 15/18] Do not upgrade Ingress when upgrading Knative * autoscale and prober tests will fail if ingress is replaced during upgrade --- test/e2e-common.sh | 54 ++++++++++++++++-------------- test/upgrade/installation/shell.go | 2 +- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index e474e4e4ff13..1e00afed06f6 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -237,29 +237,31 @@ function install_knative_serving_standard() { ko apply -f "${SERVING_RELEASE_YAML}" --selector=knative.dev/crd-install=true || return 1 fi - echo ">> Installing Ingress" - if [[ -n "${GLOO_VERSION:-}" ]]; then - install_gloo || return 1 - elif [[ -n "${KOURIER_VERSION:-}" ]]; then - install_kourier || return 1 - elif [[ -n "${AMBASSADOR_VERSION:-}" ]]; then - install_ambassador || return 1 - elif [[ -n "${CONTOUR_VERSION:-}" ]]; then - install_contour || return 1 - elif [[ -n "${KONG_VERSION:-}" ]]; then - install_kong || return 1 - else - if [[ "$1" == "HEAD" ]]; then - install_istio "./third_party/istio-latest/net-istio.yaml" || return 1 + if [[ -z "${REUSE_INGRESS:-}" ]]; then + echo ">> Installing Ingress" + if [[ -n "${GLOO_VERSION:-}" ]]; then + install_gloo || return 1 + elif [[ -n "${KOURIER_VERSION:-}" ]]; then + install_kourier || return 1 + elif [[ -n "${AMBASSADOR_VERSION:-}" ]]; then + install_ambassador || return 1 + elif [[ -n "${CONTOUR_VERSION:-}" ]]; then + install_contour || return 1 + elif [[ -n "${KONG_VERSION:-}" ]]; then + install_kong || return 1 else - # Download the latest release of net-istio. - local url="https://github.com/knative/net-istio/releases/download/${LATEST_NET_ISTIO_RELEASE_VERSION}" - local yaml="net-istio.yaml" - local YAML_NAME=${TMP_DIR}/"net-istio-${LATEST_NET_ISTIO_RELEASE_VERSION}.yaml" - wget "${url}/${yaml}" -O "${YAML_NAME}" \ - || fail_test "Unable to download latest knative/net-istio release." - echo "net-istio YAML: ${YAML_NAME}" - install_istio $YAML_NAME || return 1 + if [[ "$1" == "HEAD" ]]; then + install_istio "./third_party/istio-latest/net-istio.yaml" || return 1 + else + # Download the latest release of net-istio. + local url="https://github.com/knative/net-istio/releases/download/${LATEST_NET_ISTIO_RELEASE_VERSION}" + local yaml="net-istio.yaml" + local YAML_NAME=${TMP_DIR}/"net-istio-${LATEST_NET_ISTIO_RELEASE_VERSION}.yaml" + wget "${url}/${yaml}" -O "${YAML_NAME}" \ + || fail_test "Unable to download latest knative/net-istio release." + echo "net-istio YAML: ${YAML_NAME}" + install_istio $YAML_NAME || return 1 + fi fi fi @@ -532,9 +534,11 @@ function install_latest_release() { wait_until_batch_job_complete ${SYSTEM_NAMESPACE} } -function install_head() { - header "Installing Knative head release" - install_knative_serving || fail_test "Knative head release installation failed" +function install_head_reuse_ingress() { + header "Installing Knative head release and reusing ingress" + # Keep the existing ingress and do not upgrade it. The ingress upgrade + # makes ongoing requests fail. + REUSE_INGRESS=true install_knative_serving || fail_test "Knative head release installation failed" test_logging_config_setup wait_until_pods_running ${SYSTEM_NAMESPACE} wait_until_batch_job_complete ${SYSTEM_NAMESPACE} diff --git a/test/upgrade/installation/shell.go b/test/upgrade/installation/shell.go index c3f67f502e23..d91716ecd8ad 100644 --- a/test/upgrade/installation/shell.go +++ b/test/upgrade/installation/shell.go @@ -23,7 +23,7 @@ import ( // Head installs Knative Serving from the HEAD of the master branch. func Head() pkgupgrade.Operation { - return install("ServingHead", "install_head") + return install("ServingHead", "install_head_reuse_ingress") } // LatestRelease installs Knative Serving from the latest stable release. From a7ed4fc86443d1a93ab5e6f4284a7aa011369231 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Tue, 8 Dec 2020 10:45:01 +0100 Subject: [PATCH 16/18] Pass around func() error instead of errGroup --- test/e2e/autoscale.go | 15 ++++++++------- test/upgrade/autoscaler.go | 13 ++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index 4d9a5fa83a38..e20e9a881ccc 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -376,23 +376,24 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done // The given `duration` is how long the traffic will be generated. You must make sure that the signal // from the given `done` channel will be sent within the `duration`. func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) { - grp := AutoscaleUpToNumPods(ctx, curPods, targetPods, done, quick) - if err := grp.Wait(); err != nil { + wait := AutoscaleUpToNumPods(ctx, curPods, targetPods, done, quick) + if err := wait(); err != nil { ctx.t.Fatal(err) } } // AutoscaleUpToNumPods starts the traffic for AssertAutoscaleUpToNumPods and returns -// an error group to wait for. Starting the routines is separated from waiting for -// easy re-use in other places (e.g. upgrade tests). -func AutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) *errgroup.Group { +// a function to wait for which will return any error from test execution. +// Starting the routines is separated from waiting for easy re-use in other +// places (e.g. upgrade tests). +func AutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) func() error { // Relax the bounds to reduce the flakiness caused by sampling in the autoscaling algorithm. // Also adjust the values by the target utilization values. minPods := math.Floor(curPods/ctx.targetUtilization) - 1 maxPods := math.Ceil(math.Ceil(targetPods/ctx.targetUtilization) * 1.1) stopChan := make(chan struct{}) - grp := &errgroup.Group{} + var grp errgroup.Group grp.Go(func() error { switch ctx.metric { case autoscaling.RPS: @@ -407,5 +408,5 @@ func AutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <- return checkPodScale(ctx, targetPods, minPods, maxPods, done, quick) }) - return grp + return grp.Wait } diff --git a/test/upgrade/autoscaler.go b/test/upgrade/autoscaler.go index 39e32f97f180..18e5f7d9e50d 100644 --- a/test/upgrade/autoscaler.go +++ b/test/upgrade/autoscaler.go @@ -19,7 +19,6 @@ package upgrade import ( "time" - "golang.org/x/sync/errgroup" pkgupgrade "knative.dev/pkg/test/upgrade" "knative.dev/serving/pkg/apis/autoscaling" rtesting "knative.dev/serving/pkg/testing/v1" @@ -39,7 +38,7 @@ const ( // it is switching modes between normal and panic. func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { var ctx *e2e.TestContext - var grp *errgroup.Group + var wait func() error stopCh := make(chan time.Time) return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingTest", func(c pkgupgrade.Context) { @@ -49,14 +48,14 @@ func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { autoscaling.TargetBurstCapacityKey: "0", // Not let Activator in the path. })) ctx.SetLogger(c.Log.Infof) - grp = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) + wait = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) // Verification is done inside e2e.AssertAutoscaleUpToNumPods. // We're just giving it a signal. close(stopCh) - if err := grp.Wait(); err != nil { + if err := wait(); err != nil { c.T.Error("Error: ", err) } }, @@ -67,7 +66,7 @@ func AutoscaleSustainingTest() pkgupgrade.BackgroundOperation { // in the path a knative app scales up and sustains the scale. func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { var ctx *e2e.TestContext - var grp *errgroup.Group + var wait func() error stopCh := make(chan time.Time) return pkgupgrade.NewBackgroundVerification("AutoscaleSustainingWithTBCTest", func(c pkgupgrade.Context) { @@ -77,14 +76,14 @@ func AutoscaleSustainingWithTBCTest() pkgupgrade.BackgroundOperation { autoscaling.TargetBurstCapacityKey: "-1", // Put Activator always in the path. })) ctx.SetLogger(c.Log.Infof) - grp = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) + wait = e2e.AutoscaleUpToNumPods(ctx, curPods, targetPods, stopCh, false /* quick */) }, func(c pkgupgrade.Context) { test.EnsureTearDown(c.T, ctx.Clients(), ctx.Names()) // Verification is done inside e2e.AssertAutoscaleUpToNumPods. // We're just giving it a signal. close(stopCh) - if err := grp.Wait(); err != nil { + if err := wait(); err != nil { c.T.Error("Error: ", err) } }, From 1ff2fbe75f84a0c1e2c498ee86ec4b8151aa5337 Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Thu, 10 Dec 2020 08:50:14 +0100 Subject: [PATCH 17/18] Mark a few functions as t.Helper() --- test/e2e/autoscale.go | 2 ++ test/prober.go | 1 + 2 files changed, 3 insertions(+) diff --git a/test/e2e/autoscale.go b/test/e2e/autoscale.go index e20e9a881ccc..c3a6d1d1309b 100644 --- a/test/e2e/autoscale.go +++ b/test/e2e/autoscale.go @@ -376,6 +376,7 @@ func checkPodScale(ctx *TestContext, targetPods, minPods, maxPods float64, done // The given `duration` is how long the traffic will be generated. You must make sure that the signal // from the given `done` channel will be sent within the `duration`. func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) { + ctx.t.Helper() wait := AutoscaleUpToNumPods(ctx, curPods, targetPods, done, quick) if err := wait(); err != nil { ctx.t.Fatal(err) @@ -387,6 +388,7 @@ func AssertAutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, d // Starting the routines is separated from waiting for easy re-use in other // places (e.g. upgrade tests). func AutoscaleUpToNumPods(ctx *TestContext, curPods, targetPods float64, done <-chan time.Time, quick bool) func() error { + ctx.t.Helper() // Relax the bounds to reduce the flakiness caused by sampling in the autoscaling algorithm. // Also adjust the values by the target utilization values. minPods := math.Floor(curPods/ctx.targetUtilization) - 1 diff --git a/test/prober.go b/test/prober.go index b2d1c31e5e6c..de0702d71dc2 100644 --- a/test/prober.go +++ b/test/prober.go @@ -233,6 +233,7 @@ func RunRouteProber(logf logging.FormatLogger, clients *Clients, url *url.URL, o // against the default SLO, which requires perfect responses. // This takes `testing.T` so that it may be used in `defer`. func AssertProberDefault(t testing.TB, p Prober) { + t.Helper() AssertProberSLO(t, p, 1.0) } From 429b845dc90e41a2a30da43ab65df4769e364aee Mon Sep 17 00:00:00 2001 From: Martin Gencur Date: Mon, 4 Jan 2021 10:38:01 +0100 Subject: [PATCH 18/18] Fix codegen --- .../knative.dev/pkg/test/upgrade/functions.go | 28 +++++++++---------- .../pkg/test/upgrade/private_types.go | 28 +++++++++---------- vendor/knative.dev/pkg/test/upgrade/steps.go | 28 +++++++++---------- .../pkg/test/upgrade/suite_execution.go | 28 +++++++++---------- vendor/knative.dev/pkg/test/upgrade/types.go | 28 +++++++++---------- vendor/knative.dev/pkg/test/upgrade/vars.go | 28 +++++++++---------- 6 files changed, 84 insertions(+), 84 deletions(-) diff --git a/vendor/knative.dev/pkg/test/upgrade/functions.go b/vendor/knative.dev/pkg/test/upgrade/functions.go index 50dc637fefa4..07cd80e92ac5 100644 --- a/vendor/knative.dev/pkg/test/upgrade/functions.go +++ b/vendor/knative.dev/pkg/test/upgrade/functions.go @@ -1,18 +1,18 @@ /* - * Copyright 2020 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package upgrade diff --git a/vendor/knative.dev/pkg/test/upgrade/private_types.go b/vendor/knative.dev/pkg/test/upgrade/private_types.go index e34f090050f1..ab930e41bf62 100644 --- a/vendor/knative.dev/pkg/test/upgrade/private_types.go +++ b/vendor/knative.dev/pkg/test/upgrade/private_types.go @@ -1,18 +1,18 @@ /* - * Copyright 2020 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package upgrade diff --git a/vendor/knative.dev/pkg/test/upgrade/steps.go b/vendor/knative.dev/pkg/test/upgrade/steps.go index c50bdcfb6272..1c08038ec74e 100644 --- a/vendor/knative.dev/pkg/test/upgrade/steps.go +++ b/vendor/knative.dev/pkg/test/upgrade/steps.go @@ -1,18 +1,18 @@ /* - * Copyright 2020 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package upgrade diff --git a/vendor/knative.dev/pkg/test/upgrade/suite_execution.go b/vendor/knative.dev/pkg/test/upgrade/suite_execution.go index 70c5ea095ea7..d926d4076187 100644 --- a/vendor/knative.dev/pkg/test/upgrade/suite_execution.go +++ b/vendor/knative.dev/pkg/test/upgrade/suite_execution.go @@ -1,18 +1,18 @@ /* - * Copyright 2020 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package upgrade diff --git a/vendor/knative.dev/pkg/test/upgrade/types.go b/vendor/knative.dev/pkg/test/upgrade/types.go index afcccec332a7..e4324f4a0c8f 100644 --- a/vendor/knative.dev/pkg/test/upgrade/types.go +++ b/vendor/knative.dev/pkg/test/upgrade/types.go @@ -1,18 +1,18 @@ /* - * Copyright 2020 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package upgrade diff --git a/vendor/knative.dev/pkg/test/upgrade/vars.go b/vendor/knative.dev/pkg/test/upgrade/vars.go index 8a8ba93e11ae..6a9527b35748 100644 --- a/vendor/knative.dev/pkg/test/upgrade/vars.go +++ b/vendor/knative.dev/pkg/test/upgrade/vars.go @@ -1,18 +1,18 @@ /* - * Copyright 2020 The Knative Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +Copyright 2020 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ package upgrade