Allow traffic going outside the cluster

DEVELOPMENT.md

@@ -127,6 +127,12 @@ kubectl apply -f ./third_party/config/build/release.yaml
 
 This step includes building Knative Serving, creating and pushing developer images and deploying them to your Kubernetes cluster.
 
+First, edit [config-network.yaml](config/config-network.yaml) as instructed within the file. 
+If this file is edited and deployed after Knative Serving installation, the changes in it will be
+effective only for newly created revisions.
+
+Next, run:
+
 ```shell
 ko apply -f config/
 ```

cmd/controller/main.go

@@ -22,6 +22,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/knative/serving/pkg"
+
 	"github.com/josephburnett/k8sflag/pkg/k8sflag"
 	"github.com/knative/serving/pkg/controller"
 	"github.com/knative/serving/pkg/logging"
@@ -128,6 +130,8 @@ func main() {
 	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, time.Second*30)
 	elaInformerFactory := informers.NewSharedInformerFactory(elaClient, time.Second*30)
 	buildInformerFactory := buildinformers.NewSharedInformerFactory(buildClient, time.Second*30)
+	servingSystemInformerFactory := kubeinformers.NewFilteredSharedInformerFactory(kubeClient,
+		time.Minute*5, pkg.GetServingSystemNamespace(), nil)
 
 	revControllerConfig := revision.ControllerConfig{
 		AutoscaleConcurrencyQuantumOfTime: autoscaleConcurrencyQuantumOfTime,
@@ -148,14 +152,17 @@ func main() {
 	// Add new controllers to this array.
 	controllers := []controller.Interface{
 		configuration.NewController(kubeClient, elaClient, buildClient, kubeInformerFactory, elaInformerFactory, cfg, logger),
-		revision.NewController(kubeClient, elaClient, kubeInformerFactory, elaInformerFactory, buildInformerFactory, cfg, &revControllerConfig, logger),
-		route.NewController(kubeClient, elaClient, kubeInformerFactory, elaInformerFactory, cfg, autoscaleEnableScaleToZero, logger),
+		revision.NewController(kubeClient, elaClient, kubeInformerFactory, elaInformerFactory,
+			buildInformerFactory, servingSystemInformerFactory, cfg, &revControllerConfig, logger),
+		route.NewController(kubeClient, elaClient, kubeInformerFactory, elaInformerFactory,
+			servingSystemInformerFactory, cfg, autoscaleEnableScaleToZero, logger),
 		service.NewController(kubeClient, elaClient, kubeInformerFactory, elaInformerFactory, cfg, logger),
 	}
 
 	go kubeInformerFactory.Start(stopCh)
 	go elaInformerFactory.Start(stopCh)
 	go buildInformerFactory.Start(stopCh)
+	go servingSystemInformerFactory.Start(stopCh)
 
 	// Start all of the controllers.
 	for _, ctrlr := range controllers {

config/config-network.yaml

@@ -0,0 +1,52 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-network
+  namespace: knative-serving-system
+data:
+  # Specifies the IP ranges that Istio sidecar will intercept.
+  # Replace this with the IP ranges of your cluster (see below for some examples).
+  # Separate multiple entries with a comma.
+  # Example: "10.4.0.0/14,10.7.240.0/20"
+  # 
+  # If set to "*" Istio will intercept all traffic within
+  # the cluster as well as traffic that is going outside the cluster.
+  # Traffic going outside the cluster will be blocked unless
+  # necessary egress rules are created. 
+  #
+  # If omitted or set to "", value of global.proxy.includeIPRanges
+  # provided at Istio deployment time is used. In default Knative serving
+  # deployment, global.proxy.includeIPRanges value is set to "*".
+  # 
+  # If an invalid value is used, "" is used.
+  # 
+  # If valid set of IP address ranges are put into this value,
+  # Istio will no longer intercept traffic going to IP addresses
+  # outside the provided ranges and there is no need to specify
+  # egress rules.
+  # 
+  # To determine the IP ranges of your cluster:
+  #   IBM Cloud Private: cat cluster/config.yaml | grep service_cluster_ip_range
+  #   IBM Cloud Kubernetes Service: "172.30.0.0/16,172.20.0.0/16,10.10.10.0/24"
+  #   Google Container Engine (GKE): gcloud container clusters describe XXXXXXX --zone=XXXXXX | grep -e clusterIpv4Cidr -e servicesIpv4Cidr
+  #   Azure Container Service(ACS): "10.244.0.0/16,10.240.0.0/16"
+  #   Minikube: "10.0.0.1/24"
+  # 
+  # For more information, visit
+  # https://istio.io/docs/tasks/traffic-management/egress/
+  #
+  istio.sidecar.includeOutboundIPRanges: "*"

pkg/controller/configuration/configuration.go

@@ -71,8 +71,8 @@ func NewController(
 	revisionInformer := elaInformerFactory.Serving().V1alpha1().Revisions()
 
 	controller := &Controller{
-		Base: controller.NewBase(kubeClientSet, elaClientSet, kubeInformerFactory,
-			elaInformerFactory, informer.Informer(), controllerAgentName, "Configurations", logger),
+		Base: controller.NewBase(kubeClientSet, elaClientSet,
+			informer.Informer(), controllerAgentName, "Configurations", logger),
 		buildClientSet:  buildClientSet,
 		lister:          informer.Lister(),
 		synced:          informer.Informer().HasSynced,

pkg/controller/controller.go

@@ -22,13 +22,11 @@ import (
 
 	clientset "github.com/knative/serving/pkg/client/clientset/versioned"
 	elascheme "github.com/knative/serving/pkg/client/clientset/versioned/scheme"
-	informers "github.com/knative/serving/pkg/client/informers/externalversions"
 	"github.com/knative/serving/pkg/logging/logkey"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
-	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -57,14 +55,6 @@ type Base struct {
 	// ElaClientSet allows us to configure Ela objects
 	ElaClientSet clientset.Interface
 
-	// KubeInformerFactory provides shared informers for resources
-	// in all known API group versions
-	KubeInformerFactory kubeinformers.SharedInformerFactory
-
-	// ElaInformerFactory provides shared informers for resources
-	// in all known API group versions
-	ElaInformerFactory informers.SharedInformerFactory
-
 	// Recorder is an event recorder for recording Event resources to the
 	// Kubernetes API.
 	Recorder record.EventRecorder
@@ -89,8 +79,6 @@ type Base struct {
 func NewBase(
 	kubeClientSet kubernetes.Interface,
 	elaClientSet clientset.Interface,
-	kubeInformerFactory kubeinformers.SharedInformerFactory,
-	elaInformerFactory informers.SharedInformerFactory,
 	informer cache.SharedIndexInformer,
 	controllerAgentName string,
 	workQueueName string,
@@ -107,13 +95,11 @@ func NewBase(
 	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName})
 
 	base := &Base{
-		KubeClientSet:       kubeClientSet,
-		ElaClientSet:        elaClientSet,
-		KubeInformerFactory: kubeInformerFactory,
-		ElaInformerFactory:  elaInformerFactory,
-		Recorder:            recorder,
-		WorkQueue:           workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), workQueueName),
-		Logger:              logger,
+		KubeClientSet: kubeClientSet,
+		ElaClientSet:  elaClientSet,
+		Recorder:      recorder,
+		WorkQueue:     workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), workQueueName),
+		Logger:        logger,
 	}
 
 	// Set up an event handler for when the resource types of interest change

pkg/controller/names.go

@@ -34,6 +34,10 @@ func GetDomainConfigMapName() string {
 	return "config-domain"
 }
 
+func GetNetworkConfigMapName() string {
+	return "config-network"
+}
+
 // Various functions for naming the resources for consistency
 func GetElaNamespaceName(ns string) string {
 	// We create resources in the same namespace as the Knative Serving resources by default.

pkg/controller/revision/network_config.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package revision
+
+import (
+	"github.com/knative/serving/pkg"
+
+	"github.com/knative/serving/pkg/controller"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+const (
+	// IstioOutboundIPRangesKey is the name of the configuration entry
+	// that specifies Istio outbound ip ranges.
+	IstioOutboundIPRangesKey = "istio.sidecar.includeOutboundIPRanges"
+)
+
+// NetworkConfig contains the networking configuration defined in the
+// network config map.
+type NetworkConfig struct {
+	// IstioOutboundIPRange specifies the IP ranges to intercept
+	// by Istio sidecar.
+	IstioOutboundIPRanges string
+}
+
+// NewNetworkConfig creates a DomainConfig by reading the domain configmap from
+// the supplied client.
+func NewNetworkConfig(kubeClient kubernetes.Interface) (*NetworkConfig, error) {
+	m, err := kubeClient.CoreV1().ConfigMaps(pkg.GetServingSystemNamespace()).Get(controller.GetNetworkConfigMapName(), metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return NewNetworkConfigFromConfigMap(m), nil
+}
+
+// NewNetworkConfigFromConfigMap creates a NewNetworkConfig from the supplied ConfigMap
+func NewNetworkConfigFromConfigMap(configMap *corev1.ConfigMap) *NetworkConfig {
+	return &NetworkConfig{IstioOutboundIPRanges: configMap.Data[IstioOutboundIPRangesKey]}
+}

pkg/controller/revision/network_config_test.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2018 Google LLC.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package revision
+
+import (
+	"testing"
+
+	"github.com/knative/serving/pkg"
+	"github.com/knative/serving/pkg/controller"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	fakekubeclientset "k8s.io/client-go/kubernetes/fake"
+)
+
+func TestNewConfigMissingConfigMap(t *testing.T) {
+	_, err := NewNetworkConfig(fakekubeclientset.NewSimpleClientset())
+	if err == nil {
+		t.Error("Expected an error value when config map doesn't exist.")
+	}
+}
+
+func TestNewConfigNoEntry(t *testing.T) {
+	kubeClient := fakekubeclientset.NewSimpleClientset()
+	kubeClient.CoreV1().ConfigMaps(pkg.GetServingSystemNamespace()).Create(&corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: pkg.GetServingSystemNamespace(),
+			Name:      controller.GetNetworkConfigMapName(),
+		},
+	})
+	c, err := NewNetworkConfig(kubeClient)
+	if err != nil {
+		t.Errorf("Didn't expect an error but got %v", err)
+	} else if len(c.IstioOutboundIPRanges) > 0 {
+		t.Error("Expected an empty value when config map doesn't have the entry.")
+	}
+}
+
+func TestNewConfig(t *testing.T) {
+	kubeClient := fakekubeclientset.NewSimpleClientset()
+	want := "10.10.10.10/12"
+	kubeClient.CoreV1().ConfigMaps(pkg.GetServingSystemNamespace()).Create(&corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: pkg.GetServingSystemNamespace(),
+			Name:      controller.GetNetworkConfigMapName(),
+		},
+		Data: map[string]string{
+			IstioOutboundIPRangesKey: want,
+			"bar.com":                "selector:\n  app: bar\n  version: beta",
+		},
+	})
+	c, err := NewNetworkConfig(kubeClient)
+	if err != nil {
+		t.Errorf("Didn't expect an error but got %v", err)
+	}
+	if c.IstioOutboundIPRanges != want {
+		t.Errorf("Want %v, got %v", want, c.IstioOutboundIPRanges)
+	}
+}

pkg/controller/revision/pod.go

@@ -17,6 +17,11 @@ limitations under the License.
 package revision
 
 import (
+	"net"
+	"strings"
+
+	"go.uber.org/zap"
+
 	"github.com/knative/serving/pkg/apis/serving/v1alpha1"
 	"github.com/knative/serving/pkg/controller"
 	"github.com/knative/serving/pkg/queue"
@@ -34,8 +39,9 @@ const (
 	queueContainerCPU   = "25m"
 	fluentdContainerCPU = "75m"
 
-	fluentdConfigMapVolumeName = "configmap"
-	varLogVolumeName           = "varlog"
+	fluentdConfigMapVolumeName     = "configmap"
+	varLogVolumeName               = "varlog"
+	istioOutboundIPRangeAnnotation = "traffic.sidecar.istio.io/includeOutboundIPRanges"
 )
 
 func hasHTTPPath(p *corev1.Probe) bool {
@@ -183,7 +189,8 @@ func MakeElaPodSpec(
 }
 
 // MakeElaDeployment creates a deployment.
-func MakeElaDeployment(u *v1alpha1.Revision, namespace string) *appsv1.Deployment {
+func MakeElaDeployment(logger *zap.SugaredLogger, u *v1alpha1.Revision, namespace string,
+	networkConfig *NetworkConfig) *appsv1.Deployment {
 	rollingUpdateConfig := appsv1.RollingUpdateDeployment{
 		MaxUnavailable: &elaPodMaxUnavailable,
 		MaxSurge:       &elaPodMaxSurge,
@@ -192,6 +199,24 @@ func MakeElaDeployment(u *v1alpha1.Revision, namespace string) *appsv1.Deploymen
 	podTemplateAnnotations := MakeElaResourceAnnotations(u)
 	podTemplateAnnotations[sidecarIstioInjectAnnotation] = "true"
 
+	// Inject the IP ranges for istio sidecar configuration.
+	// We will inject this value only if all of the following are true:
+	// - the config map contains a non-empty value
+	// - the user doesn't specify this annotation in configuration's pod template
+	// - configured values are valid CIDR notation IP addresses
+	// If these conditions are not met, this value will be left untouched.
+	// * is a special value that is accepted as a valid.
+	// * intercepts calls to all IPs: in cluster as well as outside the cluster.
+	if _, ok := podTemplateAnnotations[istioOutboundIPRangeAnnotation]; !ok {
+		if len(networkConfig.IstioOutboundIPRanges) > 0 {
+			if err := validateOutboundIPRanges(networkConfig.IstioOutboundIPRanges); err != nil {
+				logger.Errorf("Failed to parse IP ranges %v. Not setting the annotation. Error: %v", networkConfig.IstioOutboundIPRanges, err)
+			} else {
+				podTemplateAnnotations[istioOutboundIPRangeAnnotation] = networkConfig.IstioOutboundIPRanges
+			}
+		}
+	}
+
 	return &appsv1.Deployment{
 		ObjectMeta: meta_v1.ObjectMeta{
 			Name:        controller.GetRevisionDeploymentName(u),
@@ -215,3 +240,17 @@ func MakeElaDeployment(u *v1alpha1.Revision, namespace string) *appsv1.Deploymen
 		},
 	}
 }
+
+func validateOutboundIPRanges(s string) error {
+	// * is a valid value
+	if s == "*" {
+		return nil
+	}
+	cidrs := strings.Split(s, ",")
+	for _, cidr := range cidrs {
+		if _, _, err := net.ParseCIDR(cidr); err != nil {
+			return err
+		}
+	}
+	return nil
+}