Expose controller-runtime metrics

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ### Added
 
+- By default the controller-runtime metrics are exposed on port 8383. This is done as part of the scaffold in the main.go file, the port can be adjusted by modifying the `metricsPort` variable. [#786](https://github.com/operator-framework/operator-sdk/pull/786)
+
 ### Changed
 
 ### Deprecated

Gopkg.toml

@@ -26,6 +26,10 @@
   name = "k8s.io/cli-runtime"
   version = "kubernetes-1.12.3"
 
+[[override]]
+   name = "k8s.io/kube-openapi"
+  revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803"
+
 [[constraint]]
   name = "sigs.k8s.io/controller-runtime"
   version = "=v0.1.8"

commands/operator-sdk/cmd/build_test.go

@@ -90,9 +90,6 @@ spec:
       containers:
         - name: memcached-operator
           image: quay.io/coreos/operator-sdk-dev:test-framework-operator
-          ports:
-          - containerPort: 60000
-            name: metrics
           command:
           - memcached-operator
           imagePullPolicy: Always

pkg/k8sutil/constants.go

@@ -27,10 +27,4 @@ const (
 	// OperatorNameEnvVar is the constant for env variable OPERATOR_NAME
 	// wich is the name of the current operator
 	OperatorNameEnvVar = "OPERATOR_NAME"
-
-	// PrometheusMetricsPort defines the port which expose prometheus metrics
-	PrometheusMetricsPort = 60000
-
-	// PrometheusMetricsPortName define the port name used in kubernetes deployment and service
-	PrometheusMetricsPortName = "metrics"
 )

pkg/k8sutil/k8sutil.go

@@ -20,9 +20,6 @@ import (
 	"os"
 	"strings"
 
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	intstr "k8s.io/apimachinery/pkg/util/intstr"
 	discovery "k8s.io/client-go/discovery"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 )
@@ -68,44 +65,6 @@ func GetOperatorName() (string, error) {
 	return operatorName, nil
 }
 
-// InitOperatorService return the static service which expose operator metrics
-func InitOperatorService() (*v1.Service, error) {
-	operatorName, err := GetOperatorName()
-	if err != nil {
-		return nil, err
-	}
-	namespace, err := GetOperatorNamespace()
-	if err != nil {
-		return nil, err
-	}
-	service := &v1.Service{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      operatorName,
-			Namespace: namespace,
-			Labels:    map[string]string{"name": operatorName},
-		},
-		TypeMeta: metav1.TypeMeta{
-			Kind:       "Service",
-			APIVersion: "v1",
-		},
-		Spec: v1.ServiceSpec{
-			Ports: []v1.ServicePort{
-				{
-					Port:     PrometheusMetricsPort,
-					Protocol: v1.ProtocolTCP,
-					TargetPort: intstr.IntOrString{
-						Type:   intstr.String,
-						StrVal: PrometheusMetricsPortName,
-					},
-					Name: PrometheusMetricsPortName,
-				},
-			},
-			Selector: map[string]string{"name": operatorName},
-		},
-	}
-	return service, nil
-}
-
 // ResourceExists returns true if the given resource kind exists
 // in the given api groupversion
 func ResourceExists(dc discovery.DiscoveryInterface, apiGroupVersion, kind string) (bool, error) {

pkg/metrics/metrics.go

@@ -16,45 +16,111 @@ package metrics
 
 import (
 	"context"
-	"net/http"
-	"strconv"
+	"fmt"
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/rest"
+	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
 )
 
 var log = logf.Log.WithName("metrics")
 
-// ExposeMetricsPort generate a Kubernetes Service to expose metrics port
-func ExposeMetricsPort() *v1.Service {
-	http.Handle("/"+k8sutil.PrometheusMetricsPortName, promhttp.Handler())
-	go http.ListenAndServe(":"+strconv.Itoa(k8sutil.PrometheusMetricsPort), nil)
+// PrometheusPortName defines the port name used in kubernetes deployment and service resources
+const PrometheusPortName = "metrics"
 
-	service, err := k8sutil.InitOperatorService()
+// ExposeMetricsPort creates a Kubernetes Service to expose the passed metrics port.
+func ExposeMetricsPort(ctx context.Context, port int32) (*v1.Service, error) {
+	// We do not need to check the validity of the port, as controller-runtime
+	// would error out and we would never get to this stage.
+	s, err := initOperatorService(port, PrometheusPortName)
 	if err != nil {
-		log.Error(err, "Failed to initialize service object for operator metrics")
-		return nil
+		if err == k8sutil.ErrNoNamespace {
+			log.Info("Skipping metrics Service creation; not running in a cluster.")
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err)
 	}
-	kubeconfig, err := config.GetConfig()
+	service, err := createService(ctx, s)
 	if err != nil {
-		panic(err)
+		return nil, fmt.Errorf("failed to create or get service for metrics: %v", err)
 	}
-	runtimeClient, err := client.New(kubeconfig, client.Options{})
+
+	return service, nil
+}
+
+func createService(ctx context.Context, s *v1.Service) (*v1.Service, error) {
+	config, err := rest.InClusterConfig()
 	if err != nil {
-		panic(err)
+		return nil, err
 	}
-	err = runtimeClient.Create(context.TODO(), service)
-	if err != nil && !errors.IsAlreadyExists(err) {
-		log.Error(err, "Failed to create service for operator metrics")
-		return nil
+
+	client, err := crclient.New(config, crclient.Options{})
+	if err != nil {
+		return nil, err
+	}
+
+	if err := client.Create(ctx, s); err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return nil, err
+		}
+		// Get existing Service and return it
+		existingService := &v1.Service{}
+		err := client.Get(ctx, types.NamespacedName{
+			Name:      s.Name,
+			Namespace: s.Namespace,
+		}, existingService)
+		if err != nil {
+			return nil, err
+		}
+		log.Info("Metrics Service object already exists", "name", existingService.Name)
+		return existingService, nil
 	}
 
-	log.Info("Metrics service created.", "ServiceName", service.Name)
-	return service
+	log.Info("Metrics Service object created", "name", s.Name)
+	return s, nil
+}
+
+// initOperatorService returns the static service which exposes specifed port.
+func initOperatorService(port int32, portName string) (*v1.Service, error) {
+	operatorName, err := k8sutil.GetOperatorName()
+	if err != nil {
+		return nil, err
+	}
+	namespace, err := k8sutil.GetOperatorNamespace()
+	if err != nil {
+		return nil, err
+	}
+	service := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      operatorName,
+			Namespace: namespace,
+			Labels:    map[string]string{"name": operatorName},
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		Spec: v1.ServiceSpec{
+			Ports: []v1.ServicePort{
+				{
+					Port:     port,
+					Protocol: v1.ProtocolTCP,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: port,
+					},
+					Name: portName,
+				},
+			},
+			Selector: map[string]string{"name": operatorName},
+		},
+	}
+	return service, nil
 }

pkg/scaffold/ansible/deploy_operator.go

@@ -57,9 +57,6 @@ spec:
         - name: {{.ProjectName}}
           # Replace this with the built image name
           image: "{{ "{{ REPLACE_IMAGE }}" }}"
-          ports:
-          - containerPort: 60000
-            name: metrics
           imagePullPolicy: "{{ "{{ pull_policy|default('Always') }}"}}"
           env:
             - name: WATCH_NAMESPACE

pkg/scaffold/cmd.go

@@ -48,6 +48,7 @@ import (
 
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
+	"github.com/operator-framework/operator-sdk/pkg/metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -56,6 +57,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
+// Change below variables to serve metrics on different host or port.
+var (
+    metricsHost = "0.0.0.0"
+    metricsPort int32 = 8383
+)
 var log = logf.Log.WithName("cmd")
 
 func printVersion() {
@@ -87,16 +93,21 @@ func main() {
 		log.Error(err, "")
 		os.Exit(1)
 	}
+
+	ctx := context.TODO()
 
 	// Become the leader before proceeding
-	err = leader.Become(context.TODO(), "{{ .ProjectName }}-lock")
+	err = leader.Become(ctx, "{{ .ProjectName }}-lock")
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
 	}
 
 	// Create a new Cmd to provide shared dependencies and start components
-	mgr, err := manager.New(cfg, manager.Options{Namespace: namespace})
+	mgr, err := manager.New(cfg, manager.Options{
+		Namespace:          namespace,
+		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
+	})	
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
@@ -116,6 +127,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Create Service object to expose the metrics port.
+	_, err = metrics.ExposeMetricsPort(ctx, metricsPort)
+	if err != nil {
+		log.Info(err.Error())
+	}
+
 	log.Info("Starting the Cmd.")
 
 	// Start the Cmd

pkg/scaffold/cmd_test.go

@@ -46,6 +46,7 @@ import (
 	"github.com/example-inc/app-operator/pkg/controller"
 	"github.com/operator-framework/operator-sdk/pkg/k8sutil"
 	"github.com/operator-framework/operator-sdk/pkg/leader"
+	"github.com/operator-framework/operator-sdk/pkg/metrics"
 	sdkVersion "github.com/operator-framework/operator-sdk/version"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
@@ -54,6 +55,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
+// Change below variables to serve metrics on different host or port.
+var (
+	metricsHost       = "0.0.0.0"
+	metricsPort int32 = 8383
+)
 var log = logf.Log.WithName("cmd")
 
 func printVersion() {
@@ -86,15 +92,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	ctx := context.TODO()
+
 	// Become the leader before proceeding
-	err = leader.Become(context.TODO(), "app-operator-lock")
+	err = leader.Become(ctx, "app-operator-lock")
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
 	}
 
 	// Create a new Cmd to provide shared dependencies and start components
-	mgr, err := manager.New(cfg, manager.Options{Namespace: namespace})
+	mgr, err := manager.New(cfg, manager.Options{
+		Namespace:          namespace,
+		MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
+	})
 	if err != nil {
 		log.Error(err, "")
 		os.Exit(1)
@@ -114,6 +125,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Create Service object to expose the metrics port.
+	_, err = metrics.ExposeMetricsPort(ctx, metricsPort)
+	if err != nil {
+		log.Info(err.Error())
+	}
+
 	log.Info("Starting the Cmd.")
 
 	// Start the Cmd

pkg/scaffold/gopkgtoml.go

@@ -92,6 +92,10 @@ required = [
   branch = "master" #osdk_branch_annotation
   # version = "=v0.4.0" #osdk_version_annotation
 
+[[override]]
+   name = "k8s.io/kube-openapi"
+  revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803"
+
 [prune]
   go-tests = true
   non-go = true

pkg/scaffold/gopkgtoml_test.go

@@ -84,6 +84,10 @@ required = [
   branch = "master" #osdk_branch_annotation
   # version = "=v0.4.0" #osdk_version_annotation
 
+[[override]]
+   name = "k8s.io/kube-openapi"
+  revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803"
+
 [prune]
   go-tests = true
   non-go = true

pkg/scaffold/operator.go

@@ -55,9 +55,6 @@ spec:
         - name: {{.ProjectName}}
           # Replace this with the built image name
           image: REPLACE_IMAGE
-          ports:
-          - containerPort: 60000
-            name: metrics
           command:
           - {{.ProjectName}}
           imagePullPolicy: Always