diff --git a/CHANGELOG.md b/CHANGELOG.md index f00f5d5b190..5165ccfd797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Added +- By default the controller-runtime metrics are exposed on port 8383. This is done as part of the scaffold in the main.go file, the port can be adjusted by modifying the `metricsPort` variable. [#786](https://github.com/operator-framework/operator-sdk/pull/786) + ### Changed ### Deprecated diff --git a/Gopkg.lock b/Gopkg.lock index 6cc55553c21..01047874080 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1446,7 +1446,6 @@ "github.com/mattbaird/jsonpatch", "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1", "github.com/pborman/uuid", - "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/sergi/go-diff/diffmatchpatch", "github.com/sirupsen/logrus", "github.com/spf13/afero", diff --git a/Gopkg.toml b/Gopkg.toml index 115803bb333..490b4739f39 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -26,6 +26,10 @@ name = "k8s.io/cli-runtime" version = "kubernetes-1.12.3" +[[override]] + name = "k8s.io/kube-openapi" + revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803" + [[constraint]] name = "sigs.k8s.io/controller-runtime" version = "=v0.1.8" diff --git a/commands/operator-sdk/cmd/build_test.go b/commands/operator-sdk/cmd/build_test.go index 63aa948e25c..2fa1b2b2896 100644 --- a/commands/operator-sdk/cmd/build_test.go +++ b/commands/operator-sdk/cmd/build_test.go @@ -90,9 +90,6 @@ spec: containers: - name: memcached-operator image: quay.io/coreos/operator-sdk-dev:test-framework-operator - ports: - - containerPort: 60000 - name: metrics command: - memcached-operator imagePullPolicy: Always diff --git a/pkg/k8sutil/constants.go b/pkg/k8sutil/constants.go index 52b6842cec7..d1842a28459 100644 --- a/pkg/k8sutil/constants.go +++ b/pkg/k8sutil/constants.go @@ -27,10 +27,4 @@ const ( // OperatorNameEnvVar is the constant for env variable OPERATOR_NAME // wich is the name of the current operator OperatorNameEnvVar = "OPERATOR_NAME" - - // PrometheusMetricsPort defines the port which expose prometheus metrics - PrometheusMetricsPort = 60000 - - // PrometheusMetricsPortName define the port name used in kubernetes deployment and service - PrometheusMetricsPortName = "metrics" ) diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 792b7d72a88..6ff8533e3b9 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -20,9 +20,6 @@ import ( "os" "strings" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - intstr "k8s.io/apimachinery/pkg/util/intstr" discovery "k8s.io/client-go/discovery" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) @@ -68,44 +65,6 @@ func GetOperatorName() (string, error) { return operatorName, nil } -// InitOperatorService return the static service which expose operator metrics -func InitOperatorService() (*v1.Service, error) { - operatorName, err := GetOperatorName() - if err != nil { - return nil, err - } - namespace, err := GetOperatorNamespace() - if err != nil { - return nil, err - } - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: operatorName, - Namespace: namespace, - Labels: map[string]string{"name": operatorName}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Port: PrometheusMetricsPort, - Protocol: v1.ProtocolTCP, - TargetPort: intstr.IntOrString{ - Type: intstr.String, - StrVal: PrometheusMetricsPortName, - }, - Name: PrometheusMetricsPortName, - }, - }, - Selector: map[string]string{"name": operatorName}, - }, - } - return service, nil -} - // ResourceExists returns true if the given resource kind exists // in the given api groupversion func ResourceExists(dc discovery.DiscoveryInterface, apiGroupVersion, kind string) (bool, error) { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 3c2844b7b92..1ea736ef580 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -16,45 +16,111 @@ package metrics import ( "context" - "net/http" - "strconv" + "fmt" "github.com/operator-framework/operator-sdk/pkg/k8sutil" - "github.com/prometheus/client_golang/prometheus/promhttp" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/rest" + crclient "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) var log = logf.Log.WithName("metrics") -// ExposeMetricsPort generate a Kubernetes Service to expose metrics port -func ExposeMetricsPort() *v1.Service { - http.Handle("/"+k8sutil.PrometheusMetricsPortName, promhttp.Handler()) - go http.ListenAndServe(":"+strconv.Itoa(k8sutil.PrometheusMetricsPort), nil) +// PrometheusPortName defines the port name used in kubernetes deployment and service resources +const PrometheusPortName = "metrics" - service, err := k8sutil.InitOperatorService() +// ExposeMetricsPort creates a Kubernetes Service to expose the passed metrics port. +func ExposeMetricsPort(ctx context.Context, port int32) (*v1.Service, error) { + // We do not need to check the validity of the port, as controller-runtime + // would error out and we would never get to this stage. + s, err := initOperatorService(port, PrometheusPortName) if err != nil { - log.Error(err, "Failed to initialize service object for operator metrics") - return nil + if err == k8sutil.ErrNoNamespace { + log.Info("Skipping metrics Service creation; not running in a cluster.") + return nil, nil + } + return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } - kubeconfig, err := config.GetConfig() + service, err := createService(ctx, s) if err != nil { - panic(err) + return nil, fmt.Errorf("failed to create or get service for metrics: %v", err) } - runtimeClient, err := client.New(kubeconfig, client.Options{}) + + return service, nil +} + +func createService(ctx context.Context, s *v1.Service) (*v1.Service, error) { + config, err := rest.InClusterConfig() if err != nil { - panic(err) + return nil, err } - err = runtimeClient.Create(context.TODO(), service) - if err != nil && !errors.IsAlreadyExists(err) { - log.Error(err, "Failed to create service for operator metrics") - return nil + + client, err := crclient.New(config, crclient.Options{}) + if err != nil { + return nil, err + } + + if err := client.Create(ctx, s); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + // Get existing Service and return it + existingService := &v1.Service{} + err := client.Get(ctx, types.NamespacedName{ + Name: s.Name, + Namespace: s.Namespace, + }, existingService) + if err != nil { + return nil, err + } + log.Info("Metrics Service object already exists", "name", existingService.Name) + return existingService, nil } - log.Info("Metrics service created.", "ServiceName", service.Name) - return service + log.Info("Metrics Service object created", "name", s.Name) + return s, nil +} + +// initOperatorService returns the static service which exposes specifed port. +func initOperatorService(port int32, portName string) (*v1.Service, error) { + operatorName, err := k8sutil.GetOperatorName() + if err != nil { + return nil, err + } + namespace, err := k8sutil.GetOperatorNamespace() + if err != nil { + return nil, err + } + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorName, + Namespace: namespace, + Labels: map[string]string{"name": operatorName}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: port, + Protocol: v1.ProtocolTCP, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: port, + }, + Name: portName, + }, + }, + Selector: map[string]string{"name": operatorName}, + }, + } + return service, nil } diff --git a/pkg/scaffold/ansible/deploy_operator.go b/pkg/scaffold/ansible/deploy_operator.go index 19fa58a80f4..cfe491d540a 100644 --- a/pkg/scaffold/ansible/deploy_operator.go +++ b/pkg/scaffold/ansible/deploy_operator.go @@ -57,9 +57,6 @@ spec: - name: {{.ProjectName}} # Replace this with the built image name image: "{{ "{{ REPLACE_IMAGE }}" }}" - ports: - - containerPort: 60000 - name: metrics imagePullPolicy: "{{ "{{ pull_policy|default('Always') }}"}}" env: - name: WATCH_NAMESPACE diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 3ee60b5abcb..15aa2f80754 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -48,6 +48,7 @@ import ( "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/operator-framework/operator-sdk/pkg/leader" + "github.com/operator-framework/operator-sdk/pkg/metrics" sdkVersion "github.com/operator-framework/operator-sdk/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -56,6 +57,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) +// Change below variables to serve metrics on different host or port. +var ( + metricsHost = "0.0.0.0" + metricsPort int32 = 8383 +) var log = logf.Log.WithName("cmd") func printVersion() { @@ -87,16 +93,21 @@ func main() { log.Error(err, "") os.Exit(1) } + + ctx := context.TODO() // Become the leader before proceeding - err = leader.Become(context.TODO(), "{{ .ProjectName }}-lock") + err = leader.Become(ctx, "{{ .ProjectName }}-lock") if err != nil { log.Error(err, "") os.Exit(1) } // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{Namespace: namespace}) + mgr, err := manager.New(cfg, manager.Options{ + Namespace: namespace, + MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), + }) if err != nil { log.Error(err, "") os.Exit(1) @@ -116,6 +127,12 @@ func main() { os.Exit(1) } + // Create Service object to expose the metrics port. + _, err = metrics.ExposeMetricsPort(ctx, metricsPort) + if err != nil { + log.Info(err.Error()) + } + log.Info("Starting the Cmd.") // Start the Cmd diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index d340cc66217..62201b83a66 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -46,6 +46,7 @@ import ( "github.com/example-inc/app-operator/pkg/controller" "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/operator-framework/operator-sdk/pkg/leader" + "github.com/operator-framework/operator-sdk/pkg/metrics" sdkVersion "github.com/operator-framework/operator-sdk/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -54,6 +55,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) +// Change below variables to serve metrics on different host or port. +var ( + metricsHost = "0.0.0.0" + metricsPort int32 = 8383 +) var log = logf.Log.WithName("cmd") func printVersion() { @@ -86,15 +92,20 @@ func main() { os.Exit(1) } + ctx := context.TODO() + // Become the leader before proceeding - err = leader.Become(context.TODO(), "app-operator-lock") + err = leader.Become(ctx, "app-operator-lock") if err != nil { log.Error(err, "") os.Exit(1) } // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{Namespace: namespace}) + mgr, err := manager.New(cfg, manager.Options{ + Namespace: namespace, + MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), + }) if err != nil { log.Error(err, "") os.Exit(1) @@ -114,6 +125,12 @@ func main() { os.Exit(1) } + // Create Service object to expose the metrics port. + _, err = metrics.ExposeMetricsPort(ctx, metricsPort) + if err != nil { + log.Info(err.Error()) + } + log.Info("Starting the Cmd.") // Start the Cmd diff --git a/pkg/scaffold/gopkgtoml.go b/pkg/scaffold/gopkgtoml.go index 582a3a59cc3..c46f070b8e5 100644 --- a/pkg/scaffold/gopkgtoml.go +++ b/pkg/scaffold/gopkgtoml.go @@ -92,6 +92,10 @@ required = [ branch = "master" #osdk_branch_annotation # version = "=v0.4.0" #osdk_version_annotation +[[override]] + name = "k8s.io/kube-openapi" + revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803" + [prune] go-tests = true non-go = true diff --git a/pkg/scaffold/gopkgtoml_test.go b/pkg/scaffold/gopkgtoml_test.go index f25d993a7e1..726b25df624 100644 --- a/pkg/scaffold/gopkgtoml_test.go +++ b/pkg/scaffold/gopkgtoml_test.go @@ -84,6 +84,10 @@ required = [ branch = "master" #osdk_branch_annotation # version = "=v0.4.0" #osdk_version_annotation +[[override]] + name = "k8s.io/kube-openapi" + revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803" + [prune] go-tests = true non-go = true diff --git a/pkg/scaffold/operator.go b/pkg/scaffold/operator.go index 12c4bae2777..b67969442f4 100644 --- a/pkg/scaffold/operator.go +++ b/pkg/scaffold/operator.go @@ -55,9 +55,6 @@ spec: - name: {{.ProjectName}} # Replace this with the built image name image: REPLACE_IMAGE - ports: - - containerPort: 60000 - name: metrics command: - {{.ProjectName}} imagePullPolicy: Always diff --git a/pkg/scaffold/operator_test.go b/pkg/scaffold/operator_test.go index 1762cc3edd7..658df0c5e6a 100644 --- a/pkg/scaffold/operator_test.go +++ b/pkg/scaffold/operator_test.go @@ -65,9 +65,6 @@ spec: - name: app-operator # Replace this with the built image name image: REPLACE_IMAGE - ports: - - containerPort: 60000 - name: metrics command: - app-operator imagePullPolicy: Always @@ -103,9 +100,6 @@ spec: - name: app-operator # Replace this with the built image name image: REPLACE_IMAGE - ports: - - containerPort: 60000 - name: metrics command: - app-operator imagePullPolicy: Always