Assorted fixes to enable chaos duck.

test/e2e-common.sh

@@ -512,6 +512,9 @@ function test_setup() {
   # Clean up kail so it doesn't interfere with job shutting down
   add_trap "kill $kail_pid || true" EXIT
 
+  echo ">> Waiting for Serving components to be running..."
+  wait_until_pods_running ${SYSTEM_NAMESPACE} || return 1
+
   local TEST_CONFIG_DIR=${TEST_DIR}/config
   echo ">> Creating test resources (${TEST_CONFIG_DIR}/)"
   ko apply ${KO_FLAGS} -f ${TEST_CONFIG_DIR}/ || return 1
@@ -525,9 +528,6 @@ function test_setup() {
   echo ">> Uploading test images..."
   ${REPO_ROOT_DIR}/test/upload-test-images.sh || return 1
 
-  echo ">> Waiting for Serving components to be running..."
-  wait_until_pods_running ${SYSTEM_NAMESPACE} || return 1
-
   echo ">> Waiting for Cert Manager components to be running..."
   wait_until_pods_running cert-manager || return 1
 

test/e2e-tests.sh

@@ -49,24 +49,19 @@ function wait_for_leader_controller() {
   return 1
 }
 
-function enable_tag_header_based_routing() {
-  echo -n "Enabling Tag Header Based Routing"
-  kubectl patch cm config-network -n "${SYSTEM_NAMESPACE}" -p '{"data":{"tagHeaderBasedRouting":"Enabled"}}'
+function toggle_feature() {
+  local FEATURE="$1"
+  local STATE="$2"
+  local CONFIG="${3:-config-features}"
+  echo -n "Setting feature ${FEATURE} to ${STATE}"
+  kubectl patch cm "${CONFIG}" -n "${SYSTEM_NAMESPACE}" -p '{"data":{"'${FEATURE}'":"'${STATE}'"}}'
+  # We don't have a good mechanism for positive handoff so sleep :(
+  echo "Waiting 10s for change to get picked up."
+  sleep 10
 }
 
-function disable_tag_header_based_routing() {
-  echo -n "Disabling Tag Header Based Routing"
-  kubectl patch cm config-network -n "${SYSTEM_NAMESPACE}" -p '{"data":{"tagHeaderBasedRouting":"Disabled"}}'
-}
-
-function enable_multi_container_feature() {
-  echo -n "Enabling Multi Container Feature Flag"
-  kubectl patch cm config-features -n "${SYSTEM_NAMESPACE}" -p '{"data":{"multi-container":"Enabled"}}'
-}
-
-function disable_multi_container_feature() {
-  echo -n "Disabling Multi Container Feature Flag"
-  kubectl patch cm config-features -n "${SYSTEM_NAMESPACE}" -p '{"data":{"multi-container":"Disabled"}}'
+function toggle_network_feature() {
+  toggle_feature "$1" "$2" config-network
 }
 
 # Script entry point.
@@ -104,7 +99,7 @@ fi
 
 
 # Keep this in sync with test/ha/ha.go
-readonly REPLICAS=2
+readonly REPLICAS=3
 readonly BUCKETS=10
 
 
@@ -172,15 +167,15 @@ if (( HTTPS )); then
   turn_off_auto_tls
 fi
 
-enable_tag_header_based_routing
-add_trap "disable_tag_header_based_routing" SIGKILL SIGTERM SIGQUIT
+toggle_network_feature tagHeaderBasedRouting Enabled
+add_trap "toggle_network_feature tagHeaderBasedRouting Disabled" SIGKILL SIGTERM SIGQUIT
 go_test_e2e -timeout=2m ./test/e2e/tagheader || failed=1
-disable_tag_header_based_routing
+toggle_network_feature tagHeaderBasedRouting Disabled
 
-enable_multi_container_feature
-add_trap "disable_multi_container_feature" SIGKILL SIGTERM SIGQUIT
+toggle_feature multi-container Enabled
+add_trap "toggle_feature multi-container Disabled" SIGKILL SIGTERM SIGQUIT
 go_test_e2e -timeout=2m ./test/e2e/multicontainer || failed=1
-disable_multi_container_feature
+toggle_feature multi-container Disabled
 
 # Certificate conformance tests must be run separately
 # because they need cert-manager specific configurations.

test/ha/activator_test.go

@@ -23,6 +23,7 @@ import (
 	"testing"
 	"time"
 
+	"knative.dev/networking/pkg/apis/networking"
 	"knative.dev/pkg/ptr"
 	"knative.dev/pkg/system"
 	"knative.dev/pkg/test/logstream"
@@ -125,7 +126,8 @@ func testActivatorHA(t *testing.T, gracePeriod *int64, slo float64) {
 		if err := pkgTest.WaitForPodDeleted(clients.KubeClient, activator.Name, system.Namespace()); err != nil {
 			t.Fatalf("Did not observe %s to actually be deleted: %v", activator.Name, err)
 		}
-		if err := pkgTest.WaitForServiceEndpoints(clients.KubeClient, resourcesScaleToZero.Revision.Name, test.ServingNamespace, test.ServingFlags.Replicas); err != nil {
+		// Check for the endpoint to appear in the activator's endpoints, since this revision may pick a subset of those endpoints.
+		if err := pkgTest.WaitForServiceEndpoints(clients.KubeClient, networking.ActivatorServiceName, system.Namespace(), test.ServingFlags.Replicas); err != nil {
 			t.Fatalf("Deployment %s failed to scale up: %v", activatorDeploymentName, err)
 		}
 		if gracePeriod != nil && *gracePeriod == 0 {

test/ha/autoscalerhpa_test.go

@@ -21,6 +21,7 @@ package ha
 import (
 	"testing"
 
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 
@@ -64,7 +65,7 @@ func TestAutoscalerHPAHANewRevision(t *testing.T) {
 
 	for _, leader := range leaders.List() {
 		if err := clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).Delete(leader,
-			&metav1.DeleteOptions{}); err != nil {
+			&metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
 			t.Fatalf("Failed to delete pod %s: %v", leader, err)
 		}
 

test/ha/controller_test.go

@@ -21,6 +21,7 @@ package ha
 import (
 	"testing"
 
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 
@@ -57,7 +58,7 @@ func TestControllerHA(t *testing.T) {
 
 	for _, leader := range leaders.List() {
 		if err := clients.KubeClient.Kube.CoreV1().Pods(system.Namespace()).Delete(leader,
-			&metav1.DeleteOptions{}); err != nil {
+			&metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
 			t.Fatalf("Failed to delete pod %s: %v", leader, err)
 		}