diff --git a/test/e2e/autoscale_test.go b/test/e2e/autoscale_test.go
index 4fea4c240618..54b51291641a 100644
--- a/test/e2e/autoscale_test.go
+++ b/test/e2e/autoscale_test.go
@@ -40,8 +40,7 @@ const (
 )
 
 var (
-	initialScaleToZeroThreshold   string
-	initialScaleToZeroGracePeriod string
+	scaleToZeroThreshold time.Duration
 )
 
 func isDeploymentScaledUp() func(d *v1beta1.Deployment) (bool, error) {
@@ -118,45 +117,22 @@ func generateTraffic(clients *test.Clients, logger *logging.BaseLogger, concurre
 	return nil
 }
 
-func getAutoscalerConfigMap(clients *test.Clients) (*v1.ConfigMap, error) {
-	return test.GetConfigMap(clients.KubeClient).Get("config-autoscaler", metav1.GetOptions{})
-}
-
-func setScaleToZeroThreshold(clients *test.Clients, threshold string, gracePeriod string) error {
-	configMap, err := getAutoscalerConfigMap(clients)
-	if err != nil {
-		return err
-	}
-	configMap.Data["scale-to-zero-threshold"] = threshold
-	configMap.Data["scale-to-zero-grace-period"] = gracePeriod
-	_, err = test.GetConfigMap(clients.KubeClient).Update(configMap)
-	return err
-}
-
 func setup(t *testing.T, logger *logging.BaseLogger) *test.Clients {
 	clients := Setup(t)
 
-	configMap, err := getAutoscalerConfigMap(clients)
+	configMap, err := test.GetConfigMap(clients.KubeClient).Get("config-autoscaler", metav1.GetOptions{})
 	if err != nil {
-		logger.Infof("Unable to retrieve the autoscale configMap. Assuming a ScaleToZero value of '5m'. %v", err)
-		initialScaleToZeroThreshold = "5m"
-		initialScaleToZeroGracePeriod = "2m"
-	} else {
-		initialScaleToZeroThreshold = configMap.Data["scale-to-zero-threshold"]
-		initialScaleToZeroGracePeriod = configMap.Data["scale-to-zero-grace-period"]
+		t.Fatalf("Unable to get autoscaler config map: %v", err)
 	}
-
-	err = setScaleToZeroThreshold(clients, "1m", "30s")
+	scaleToZeroThreshold, err = time.ParseDuration(configMap.Data["scale-to-zero-threshold"])
 	if err != nil {
-		t.Fatalf(`Unable to set ScaleToZeroThreshold to '1m'. This will
-		          cause the test to time out. Failing fast instead. %v`, err)
+		t.Fatalf("Unable to parse scale-to-zero-threshold as duration: %v", err)
 	}
 
 	return clients
 }
 
 func tearDown(clients *test.Clients, names test.ResourceNames, logger *logging.BaseLogger) {
-	setScaleToZeroThreshold(clients, initialScaleToZeroThreshold, initialScaleToZeroGracePeriod)
 	TearDown(clients, names, logger)
 }
 
@@ -232,7 +208,8 @@ func TestAutoscaleUpDownUp(t *testing.T) {
 		clients.KubeClient,
 		deploymentName,
 		isDeploymentScaledUp(),
-		"DeploymentIsScaledUp")
+		"DeploymentIsScaledUp",
+		2*time.Minute)
 	if err != nil {
 		logger.Fatalf(`Unable to observe the Deployment named %s scaling
 			   up. %s`, deploymentName, err)
@@ -241,15 +218,13 @@ func TestAutoscaleUpDownUp(t *testing.T) {
 	logger.Infof(`The autoscaler successfully scales down when devoid of
 		    traffic.`)
 
-	logger.Infof(`Manually setting ScaleToZeroThreshold to '1m' to facilitate
-		    faster testing.`)
-
 	logger.Infof("Waiting for scale to zero")
 	err = test.WaitForDeploymentState(
 		clients.KubeClient,
 		deploymentName,
 		isDeploymentScaledToZero(),
-		"DeploymentScaledToZero")
+		"DeploymentScaledToZero",
+		scaleToZeroThreshold+2*time.Minute)
 	if err != nil {
 		logger.Fatalf(`Unable to observe the Deployment named %s scaling
 		           down. %s`, deploymentName, err)
@@ -283,7 +258,8 @@ func TestAutoscaleUpDownUp(t *testing.T) {
 		clients.KubeClient,
 		deploymentName,
 		isDeploymentScaledUp(),
-		"DeploymentScaledUp")
+		"DeploymentScaledUp",
+		2*time.Minute)
 	if err != nil {
 		logger.Fatalf(`Unable to observe the Deployment named %s scaling
 			   up. %s`, deploymentName, err)
diff --git a/test/kube_checks.go b/test/kube_checks.go
index 1c8e7f0f5e00..62eeb25b1d96 100644
--- a/test/kube_checks.go
+++ b/test/kube_checks.go
@@ -22,6 +22,7 @@ package test
 import (
 	"context"
 	"fmt"
+	"time"
 
 	pkgTest "github.com/knative/pkg/test"
 	"go.opencensus.io/trace"
@@ -35,7 +36,7 @@ import (
 // from client every interval until inState returns `true` indicating it
 // is done, returns an error or timeout. desc will be used to name the metric
 // that is emitted to track how long it took for name to get into the state checked by inState.
-func WaitForDeploymentState(client *pkgTest.KubeClient, name string, inState func(d *apiv1beta1.Deployment) (bool, error), desc string) error {
+func WaitForDeploymentState(client *pkgTest.KubeClient, name string, inState func(d *apiv1beta1.Deployment) (bool, error), desc string, timeout time.Duration) error {
 	d := client.Kube.ExtensionsV1beta1().Deployments(ServingNamespace)
 	metricName := fmt.Sprintf("WaitForDeploymentState/%s/%s", name, desc)
 	_, span := trace.StartSpan(context.Background(), metricName)