From 9e42d3f72565e8f1ddbb2845474cd91f4edf6df2 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 16:22:00 +0100 Subject: [PATCH 01/34] pkg/metrics/metrics.go: Use controller-runtime metrics --- pkg/metrics/metrics.go | 40 ++++++++++++++-------------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 3c2844b7b92..68605b60028 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -15,46 +15,34 @@ package metrics import ( - "context" - "net/http" + "fmt" + "net" "strconv" "github.com/operator-framework/operator-sdk/pkg/k8sutil" - "github.com/prometheus/client_golang/prometheus/promhttp" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) var log = logf.Log.WithName("metrics") -// ExposeMetricsPort generate a Kubernetes Service to expose metrics port -func ExposeMetricsPort() *v1.Service { - http.Handle("/"+k8sutil.PrometheusMetricsPortName, promhttp.Handler()) - go http.ListenAndServe(":"+strconv.Itoa(k8sutil.PrometheusMetricsPort), nil) - - service, err := k8sutil.InitOperatorService() +// ExposeMetricsPort generates a Kubernetes Service to expose the metrics port +func ExposeMetricsPort(address string) (*v1.Service, error) { + // Split out port from address, to pass to Service object. + // We do not need to check the validity of the port, as controller-runtime + // would error out and we would never get to this stage. + _, port, err := net.SplitHostPort(address) if err != nil { - log.Error(err, "Failed to initialize service object for operator metrics") - return nil + return nil, fmt.Errorf("failed to split metrics address %s: %v", address, err) } - kubeconfig, err := config.GetConfig() + port64, err := strconv.ParseInt(port, 0, 32) if err != nil { - panic(err) + return nil, fmt.Errorf("failed to split metrics address %s: %v", address, err) } - runtimeClient, err := client.New(kubeconfig, client.Options{}) + service, err := k8sutil.InitOperatorService(int32(port64), PrometheusPortName) if err != nil { - panic(err) - } - err = runtimeClient.Create(context.TODO(), service) - if err != nil && !errors.IsAlreadyExists(err) { - log.Error(err, "Failed to create service for operator metrics") - return nil + return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } - - log.Info("Metrics service created.", "ServiceName", service.Name) - return service + return service, nil } From cfc26da4f869dcd478a4659b1941d84ad56c28e5 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 16:22:42 +0100 Subject: [PATCH 02/34] pkg/k8sutil/k8sutil.go: Make function take in args --- pkg/k8sutil/k8sutil.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 792b7d72a88..7c560be6a60 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -68,8 +68,8 @@ func GetOperatorName() (string, error) { return operatorName, nil } -// InitOperatorService return the static service which expose operator metrics -func InitOperatorService() (*v1.Service, error) { +// InitOperatorService returns the static service which exposes specifed port. +func InitOperatorService(port int32, portName string) (*v1.Service, error) { operatorName, err := GetOperatorName() if err != nil { return nil, err @@ -91,13 +91,13 @@ func InitOperatorService() (*v1.Service, error) { Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ { - Port: PrometheusMetricsPort, + Port: port, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{ Type: intstr.String, - StrVal: PrometheusMetricsPortName, + StrVal: portName, }, - Name: PrometheusMetricsPortName, + Name: portName, }, }, Selector: map[string]string{"name": operatorName}, From 6030574f95e75ad106f981c734ca4c80d9a128f6 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 16:23:22 +0100 Subject: [PATCH 03/34] pkg/*: Move metric specific constant --- pkg/k8sutil/constants.go | 6 ------ pkg/metrics/constants.go | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 pkg/metrics/constants.go diff --git a/pkg/k8sutil/constants.go b/pkg/k8sutil/constants.go index 52b6842cec7..d1842a28459 100644 --- a/pkg/k8sutil/constants.go +++ b/pkg/k8sutil/constants.go @@ -27,10 +27,4 @@ const ( // OperatorNameEnvVar is the constant for env variable OPERATOR_NAME // wich is the name of the current operator OperatorNameEnvVar = "OPERATOR_NAME" - - // PrometheusMetricsPort defines the port which expose prometheus metrics - PrometheusMetricsPort = 60000 - - // PrometheusMetricsPortName define the port name used in kubernetes deployment and service - PrometheusMetricsPortName = "metrics" ) diff --git a/pkg/metrics/constants.go b/pkg/metrics/constants.go new file mode 100644 index 00000000000..780ffe690c6 --- /dev/null +++ b/pkg/metrics/constants.go @@ -0,0 +1,20 @@ +// Copyright 2018 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +const ( + // PrometheusPortName defines the port name used in kubernetes deployment and service resources + PrometheusPortName = "metrics" +) From c4e89c3d4564eba360d9ca9b469e160a155397f0 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 16:24:51 +0100 Subject: [PATCH 04/34] pkg/scaffold/operator.go: Remove metrics port --- pkg/scaffold/ansible/operator.go | 3 --- pkg/scaffold/operator.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/pkg/scaffold/ansible/operator.go b/pkg/scaffold/ansible/operator.go index 869f67c8a09..f7e35608e15 100644 --- a/pkg/scaffold/ansible/operator.go +++ b/pkg/scaffold/ansible/operator.go @@ -54,9 +54,6 @@ spec: - name: {{.ProjectName}} # Replace this with the built image name image: REPLACE_IMAGE - ports: - - containerPort: 60000 - name: metrics imagePullPolicy: Always env: - name: WATCH_NAMESPACE diff --git a/pkg/scaffold/operator.go b/pkg/scaffold/operator.go index df3cf2e6349..155a33b5d3f 100644 --- a/pkg/scaffold/operator.go +++ b/pkg/scaffold/operator.go @@ -55,9 +55,6 @@ spec: - name: {{.ProjectName}} # Replace this with the built image name image: REPLACE_IMAGE - ports: - - containerPort: 60000 - name: metrics command: - {{.ProjectName}} imagePullPolicy: Always From 557c2b21b28d949db1d8a3254278db044c1b322b Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 16:25:38 +0100 Subject: [PATCH 05/34] pkg/scaffold/cmd.go: Add metrics to operators --- pkg/scaffold/cmd.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 521b051d43b..8acddee7f3e 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -48,6 +48,7 @@ import ( "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/operator-framework/operator-sdk/pkg/leader" + "github.com/operator-framework/operator-sdk/pkg/metrics" "github.com/operator-framework/operator-sdk/pkg/ready" sdkVersion "github.com/operator-framework/operator-sdk/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -58,6 +59,8 @@ import ( ) var log = logf.Log.WithName("cmd") +// Change metricsAddress to serve metrics on different host and port. +var metricsAddress string = ":8080" func printVersion() { log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) @@ -101,7 +104,7 @@ func main() { defer r.Unset() // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{Namespace: namespace}) + mgr, err := manager.New(cfg, manager.Options{Namespace: namespace, MetricsBindAddress: metricsAddress}) if err != nil { log.Error(err, "") os.Exit(1) @@ -121,6 +124,15 @@ func main() { os.Exit(1) } + // Expose metrics by creating a Service object + s, err := metrics.ExposeMetricsPort(metricsAddress) + if err != nil { + log.Error(err, "") + os.Exit(1) + } + client := mgr.GetClient() + client.Create(context.TODO(), s) + log.Info("Starting the Cmd.") // Start the Cmd From 19a2cb12f0268555f0c0ece8012feedeba1d8e48 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 16:47:43 +0100 Subject: [PATCH 06/34] pkg/scaffold/cmd_test.go: Adjust test --- pkg/scaffold/cmd_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 7242fa2a9a3..4c56e04230c 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -46,6 +46,7 @@ import ( "github.com/example-inc/app-operator/pkg/controller" "github.com/operator-framework/operator-sdk/pkg/k8sutil" "github.com/operator-framework/operator-sdk/pkg/leader" + "github.com/operator-framework/operator-sdk/pkg/metrics" "github.com/operator-framework/operator-sdk/pkg/ready" sdkVersion "github.com/operator-framework/operator-sdk/version" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" @@ -56,6 +57,8 @@ import ( ) var log = logf.Log.WithName("cmd") +// Change metricsAddress to serve metrics on different host and port. +var metricsAddress string = ":8080" func printVersion() { log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) @@ -118,6 +121,15 @@ func main() { log.Error(err, "") os.Exit(1) } + + // Expose metrics by creating a Service object + s, err := metrics.ExposeMetricsPort(metricsAddress) + if err != nil { + log.Error(err, "") + os.Exit(1) + } + client := mgr.GetClient() + client.Create(context.TODO(), s) log.Info("Starting the Cmd.") From 0a64797e08df22af948b12e6c42867d381593926 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 17:42:34 +0100 Subject: [PATCH 07/34] pkg/scaffold/cmd_test.go: Adjust test --- pkg/scaffold/cmd_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 4c56e04230c..f3d13200a3d 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -102,7 +102,7 @@ func main() { defer r.Unset() // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{Namespace: namespace}) + mgr, err := manager.New(cfg, manager.Options{Namespace: namespace, MetricsBindAddress: metricsAddress}) if err != nil { log.Error(err, "") os.Exit(1) From b5a0e7b44ab40f0c09bae5669ada0cfe13e2abf5 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 29 Nov 2018 17:43:35 +0100 Subject: [PATCH 08/34] pkg/scaffold/operator_test.go: Adjust test --- pkg/scaffold/operator_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/scaffold/operator_test.go b/pkg/scaffold/operator_test.go index 289983c32f6..b781242b0ce 100644 --- a/pkg/scaffold/operator_test.go +++ b/pkg/scaffold/operator_test.go @@ -65,9 +65,6 @@ spec: - name: app-operator # Replace this with the built image name image: REPLACE_IMAGE - ports: - - containerPort: 60000 - name: metrics command: - app-operator imagePullPolicy: Always From e6b6f855328cd682ff7f4aeb087b57e1ea60c759 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Fri, 30 Nov 2018 10:39:13 +0100 Subject: [PATCH 09/34] Update pkg/metrics/metrics.go Co-Authored-By: LiliC --- pkg/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 68605b60028..2385dea4b06 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -38,7 +38,7 @@ func ExposeMetricsPort(address string) (*v1.Service, error) { } port64, err := strconv.ParseInt(port, 0, 32) if err != nil { - return nil, fmt.Errorf("failed to split metrics address %s: %v", address, err) + return nil, fmt.Errorf("failed to parse metrics address %s: %v", address, err) } service, err := k8sutil.InitOperatorService(int32(port64), PrometheusPortName) if err != nil { From 4d0706a05ac11b653d1671ca20ed122375b43925 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Fri, 30 Nov 2018 10:52:08 +0100 Subject: [PATCH 10/34] *: Adjust tests --- commands/operator-sdk/cmd/build_test.go | 3 --- pkg/scaffold/operator_test.go | 3 --- 2 files changed, 6 deletions(-) diff --git a/commands/operator-sdk/cmd/build_test.go b/commands/operator-sdk/cmd/build_test.go index 63aa948e25c..2fa1b2b2896 100644 --- a/commands/operator-sdk/cmd/build_test.go +++ b/commands/operator-sdk/cmd/build_test.go @@ -90,9 +90,6 @@ spec: containers: - name: memcached-operator image: quay.io/coreos/operator-sdk-dev:test-framework-operator - ports: - - containerPort: 60000 - name: metrics command: - memcached-operator imagePullPolicy: Always diff --git a/pkg/scaffold/operator_test.go b/pkg/scaffold/operator_test.go index b781242b0ce..7984eb1213c 100644 --- a/pkg/scaffold/operator_test.go +++ b/pkg/scaffold/operator_test.go @@ -108,9 +108,6 @@ spec: - name: app-operator # Replace this with the built image name image: REPLACE_IMAGE - ports: - - containerPort: 60000 - name: metrics command: - app-operator imagePullPolicy: Always From 509442c5ce7a5007de55c9a74dfc91e3ef9aa6e7 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Fri, 30 Nov 2018 11:39:58 +0100 Subject: [PATCH 11/34] pkg/scaffold: Adjust test --- pkg/scaffold/cmd.go | 2 +- pkg/scaffold/cmd_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 8acddee7f3e..986e9d16a43 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -58,9 +58,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) -var log = logf.Log.WithName("cmd") // Change metricsAddress to serve metrics on different host and port. var metricsAddress string = ":8080" +var log = logf.Log.WithName("cmd") func printVersion() { log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index f3d13200a3d..8247105df9b 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -56,9 +56,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) -var log = logf.Log.WithName("cmd") // Change metricsAddress to serve metrics on different host and port. var metricsAddress string = ":8080" +var log = logf.Log.WithName("cmd") func printVersion() { log.Info(fmt.Sprintf("Go Version: %s", runtime.Version())) @@ -121,7 +121,7 @@ func main() { log.Error(err, "") os.Exit(1) } - + // Expose metrics by creating a Service object s, err := metrics.ExposeMetricsPort(metricsAddress) if err != nil { From 02a27cd2f648365a9937d06a4113388abbe17e91 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 3 Dec 2018 14:42:54 +0100 Subject: [PATCH 12/34] pkg/scaffold: Use different default port --- pkg/scaffold/cmd.go | 2 +- pkg/scaffold/cmd_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 986e9d16a43..2f480442635 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -59,7 +59,7 @@ import ( ) // Change metricsAddress to serve metrics on different host and port. -var metricsAddress string = ":8080" +var metricsAddress string = ":8383" var log = logf.Log.WithName("cmd") func printVersion() { diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 8247105df9b..3170c9eb488 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -57,7 +57,7 @@ import ( ) // Change metricsAddress to serve metrics on different host and port. -var metricsAddress string = ":8080" +var metricsAddress string = ":8383" var log = logf.Log.WithName("cmd") func printVersion() { From aecbeb5245ba8bb85772f81bc3b1c62b5b56000d Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Wed, 5 Dec 2018 10:13:49 +0100 Subject: [PATCH 13/34] pkg/scaffold: Add error check --- pkg/scaffold/cmd.go | 5 ++++- pkg/scaffold/cmd_test.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 2f480442635..1ce1b84f5c6 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -131,7 +131,10 @@ func main() { os.Exit(1) } client := mgr.GetClient() - client.Create(context.TODO(), s) + if err := client.Create(context.TODO(), s); err != nil { + log.Error(err, "") + os.Exit(1) + } log.Info("Starting the Cmd.") diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 3170c9eb488..da8e6e09bc4 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -129,7 +129,10 @@ func main() { os.Exit(1) } client := mgr.GetClient() - client.Create(context.TODO(), s) + if err := client.Create(context.TODO(), s); err != nil { + log.Error(err, "") + os.Exit(1) + } log.Info("Starting the Cmd.") From ce72d9514eae8c94d5e2bb100a662bbc460ac021 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Wed, 5 Dec 2018 10:16:05 +0100 Subject: [PATCH 14/34] pkg/metrics: Remove constant from constants file --- pkg/metrics/constants.go | 20 -------------------- pkg/metrics/metrics.go | 3 +++ 2 files changed, 3 insertions(+), 20 deletions(-) delete mode 100644 pkg/metrics/constants.go diff --git a/pkg/metrics/constants.go b/pkg/metrics/constants.go deleted file mode 100644 index 780ffe690c6..00000000000 --- a/pkg/metrics/constants.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2018 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package metrics - -const ( - // PrometheusPortName defines the port name used in kubernetes deployment and service resources - PrometheusPortName = "metrics" -) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 2385dea4b06..00931cb0c31 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -27,6 +27,9 @@ import ( var log = logf.Log.WithName("metrics") +// PrometheusPortName defines the port name used in kubernetes deployment and service resources +const PrometheusPortName = "metrics" + // ExposeMetricsPort generates a Kubernetes Service to expose the metrics port func ExposeMetricsPort(address string) (*v1.Service, error) { // Split out port from address, to pass to Service object. From 1ab7fca564961e35c4a3695987850438193a3ba8 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 6 Dec 2018 14:24:21 +0100 Subject: [PATCH 15/34] pkg/k8sutil: Use port number --- pkg/k8sutil/k8sutil.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index 7c560be6a60..ad40b904ab2 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -94,8 +94,8 @@ func InitOperatorService(port int32, portName string) (*v1.Service, error) { Port: port, Protocol: v1.ProtocolTCP, TargetPort: intstr.IntOrString{ - Type: intstr.String, - StrVal: portName, + Type: intstr.Int, + IntVal: port, }, Name: portName, }, From 48364adab3dc3d19b86dee50a93e266751e5ca93 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 10 Dec 2018 12:09:33 +0100 Subject: [PATCH 16/34] Gopkg.lock: Run dep ensure --- Gopkg.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Gopkg.lock b/Gopkg.lock index 688b67b94a9..a16b3224694 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1427,7 +1427,6 @@ "github.com/martinlindhe/base36", "github.com/mattbaird/jsonpatch", "github.com/pborman/uuid", - "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/sergi/go-diff/diffmatchpatch", "github.com/sirupsen/logrus", "github.com/spf13/afero", From 76312aceac22e64ff6526cdcfdbb85edd1f36a11 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 10 Dec 2018 13:47:35 +0100 Subject: [PATCH 17/34] pkg/scaffold/cmd.go: Skip erroring out on Service If the Service could not be created we just log the error instead of erroring out and exiting. This error could occur when there is more than one pods such as with leader election. In that case the new pod will expose the metrics via the already created Service object. --- pkg/scaffold/cmd.go | 3 ++- pkg/scaffold/cmd_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 1ce1b84f5c6..16cbb118ad5 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -51,6 +51,7 @@ import ( "github.com/operator-framework/operator-sdk/pkg/metrics" "github.com/operator-framework/operator-sdk/pkg/ready" sdkVersion "github.com/operator-framework/operator-sdk/version" + apierrors "k8s.io/apimachinery/pkg/api/errors" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -131,7 +132,7 @@ func main() { os.Exit(1) } client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil { + if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { log.Error(err, "") os.Exit(1) } diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index da8e6e09bc4..197a68f1354 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -49,6 +49,7 @@ import ( "github.com/operator-framework/operator-sdk/pkg/metrics" "github.com/operator-framework/operator-sdk/pkg/ready" sdkVersion "github.com/operator-framework/operator-sdk/version" + apierrors "k8s.io/apimachinery/pkg/api/errors" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -129,7 +130,7 @@ func main() { os.Exit(1) } client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil { + if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { log.Error(err, "") os.Exit(1) } From 2a3eb48f85801c79327f9611261a30d7f5313595 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 7 Jan 2019 15:34:50 +0100 Subject: [PATCH 18/34] pkg/scaffold: Skip exit if metrics fail to be setup --- pkg/scaffold/cmd.go | 11 +++++------ pkg/scaffold/cmd_test.go | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 16cbb118ad5..9bc2d1a6756 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -129,14 +129,13 @@ func main() { s, err := metrics.ExposeMetricsPort(metricsAddress) if err != nil { log.Error(err, "") - os.Exit(1) } - client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { - log.Error(err, "") - os.Exit(1) + if err == nil { + client := mgr.GetClient() + if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { + log.Error(err, "") + } } - log.Info("Starting the Cmd.") // Start the Cmd diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 197a68f1354..3f40807ce86 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -127,14 +127,13 @@ func main() { s, err := metrics.ExposeMetricsPort(metricsAddress) if err != nil { log.Error(err, "") - os.Exit(1) } - client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { - log.Error(err, "") - os.Exit(1) + if err == nil { + client := mgr.GetClient() + if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { + log.Error(err, "") + } } - log.Info("Starting the Cmd.") // Start the Cmd From 84a221338e8cc8a9362c6dc32206a45ac2e97cb8 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 8 Jan 2019 12:30:46 +0100 Subject: [PATCH 19/34] Gopkg.toml: Add override for kube-openapi --- Gopkg.toml | 4 ++++ pkg/scaffold/gopkgtoml.go | 4 ++++ pkg/scaffold/gopkgtoml_test.go | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/Gopkg.toml b/Gopkg.toml index f6833f71502..f29aa76d8d7 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -60,3 +60,7 @@ [[override]] name = "github.com/docker/docker" revision = "a9fbbdc8dd8794b20af358382ab780559bca589d" + +[[override]] + name = "k8s.io/kube-openapi" + revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803" diff --git a/pkg/scaffold/gopkgtoml.go b/pkg/scaffold/gopkgtoml.go index 1da9438429e..6b350be1499 100644 --- a/pkg/scaffold/gopkgtoml.go +++ b/pkg/scaffold/gopkgtoml.go @@ -92,6 +92,10 @@ required = [ branch = "master" #osdk_branch_annotation # version = "=v0.3.0" #osdk_version_annotation +[[override]] + name = "k8s.io/kube-openapi" + revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803" + [prune] go-tests = true non-go = true diff --git a/pkg/scaffold/gopkgtoml_test.go b/pkg/scaffold/gopkgtoml_test.go index 34daf659b3b..6c2d12c3185 100644 --- a/pkg/scaffold/gopkgtoml_test.go +++ b/pkg/scaffold/gopkgtoml_test.go @@ -84,6 +84,10 @@ required = [ branch = "master" #osdk_branch_annotation # version = "=v0.3.0" #osdk_version_annotation +[[override]] + name = "k8s.io/kube-openapi" + revision = "0cf8f7e6ed1d2e3d47d02e3b6e559369af24d803" + [prune] go-tests = true non-go = true From a71a3b2a08457091129f5673f2efc1ab8935006a Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 8 Jan 2019 14:35:20 +0100 Subject: [PATCH 20/34] pkg/scaffold: Refactor --- pkg/scaffold/cmd.go | 27 ++++++++++++++++----------- pkg/scaffold/cmd_test.go | 27 ++++++++++++++++----------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 9bc2d1a6756..ba4bca84d31 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -125,17 +125,8 @@ func main() { os.Exit(1) } - // Expose metrics by creating a Service object - s, err := metrics.ExposeMetricsPort(metricsAddress) - if err != nil { - log.Error(err, "") - } - if err == nil { - client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { - log.Error(err, "") - } - } + setupMetrics(mgr) + log.Info("Starting the Cmd.") // Start the Cmd @@ -144,4 +135,18 @@ func main() { os.Exit(1) } } + +func setupMetrics(mgr manager.Manager) { + // Expose metrics by creating a Service object + s, err := metrics.ExposeMetricsPort(metricsAddress) + if err != nil { + log.Info(err.Error()) + return + } + client := mgr.GetClient() + if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { + log.Info(err.Error()) + return + } +} ` diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 3f40807ce86..72ca1a47607 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -123,17 +123,8 @@ func main() { os.Exit(1) } - // Expose metrics by creating a Service object - s, err := metrics.ExposeMetricsPort(metricsAddress) - if err != nil { - log.Error(err, "") - } - if err == nil { - client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { - log.Error(err, "") - } - } + setupMetrics(mgr) + log.Info("Starting the Cmd.") // Start the Cmd @@ -142,4 +133,18 @@ func main() { os.Exit(1) } } + +func setupMetrics(mgr manager.Manager) { + // Expose metrics by creating a Service object + s, err := metrics.ExposeMetricsPort(metricsAddress) + if err != nil { + log.Info(err.Error()) + return + } + client := mgr.GetClient() + if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { + log.Info(err.Error()) + return + } +} ` From 2adb7f68cf7428dadf627c238cf1c92add4e7cc7 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 10 Jan 2019 15:31:50 +0100 Subject: [PATCH 21/34] pkg/*: Move InitOperatorService helper to metrics pkg and make it private. --- pkg/k8sutil/k8sutil.go | 41 ----------------------------------------- pkg/metrics/metrics.go | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go index ad40b904ab2..6ff8533e3b9 100644 --- a/pkg/k8sutil/k8sutil.go +++ b/pkg/k8sutil/k8sutil.go @@ -20,9 +20,6 @@ import ( "os" "strings" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - intstr "k8s.io/apimachinery/pkg/util/intstr" discovery "k8s.io/client-go/discovery" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) @@ -68,44 +65,6 @@ func GetOperatorName() (string, error) { return operatorName, nil } -// InitOperatorService returns the static service which exposes specifed port. -func InitOperatorService(port int32, portName string) (*v1.Service, error) { - operatorName, err := GetOperatorName() - if err != nil { - return nil, err - } - namespace, err := GetOperatorNamespace() - if err != nil { - return nil, err - } - service := &v1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: operatorName, - Namespace: namespace, - Labels: map[string]string{"name": operatorName}, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "Service", - APIVersion: "v1", - }, - Spec: v1.ServiceSpec{ - Ports: []v1.ServicePort{ - { - Port: port, - Protocol: v1.ProtocolTCP, - TargetPort: intstr.IntOrString{ - Type: intstr.Int, - IntVal: port, - }, - Name: portName, - }, - }, - Selector: map[string]string{"name": operatorName}, - }, - } - return service, nil -} - // ResourceExists returns true if the given resource kind exists // in the given api groupversion func ResourceExists(dc discovery.DiscoveryInterface, apiGroupVersion, kind string) (bool, error) { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 00931cb0c31..33b3fb8267f 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -22,6 +22,8 @@ import ( "github.com/operator-framework/operator-sdk/pkg/k8sutil" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) @@ -43,9 +45,47 @@ func ExposeMetricsPort(address string) (*v1.Service, error) { if err != nil { return nil, fmt.Errorf("failed to parse metrics address %s: %v", address, err) } - service, err := k8sutil.InitOperatorService(int32(port64), PrometheusPortName) + service, err := initOperatorService(int32(port64), PrometheusPortName) if err != nil { return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } return service, nil } + +// initOperatorService returns the static service which exposes specifed port. +func initOperatorService(port int32, portName string) (*v1.Service, error) { + operatorName, err := k8sutil.GetOperatorName() + if err != nil { + return nil, err + } + namespace, err := k8sutil.GetOperatorNamespace() + if err != nil { + return nil, err + } + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorName, + Namespace: namespace, + Labels: map[string]string{"name": operatorName}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: port, + Protocol: v1.ProtocolTCP, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: port, + }, + Name: portName, + }, + }, + Selector: map[string]string{"name": operatorName}, + }, + } + return service, nil +} From 18cc5ceec102ddec663eec7a2c8336b3cf1e504f Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 21 Jan 2019 14:12:52 +0100 Subject: [PATCH 22/34] pkg/*: Move Service creation logic to metrics helper --- pkg/metrics/metrics.go | 21 +++++++++++++++++---- pkg/scaffold/cmd.go | 21 +++++---------------- pkg/scaffold/cmd_test.go | 21 +++++---------------- 3 files changed, 27 insertions(+), 36 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 33b3fb8267f..630aa10176d 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -15,6 +15,7 @@ package metrics import ( + "context" "fmt" "net" "strconv" @@ -24,6 +25,7 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/manager" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) @@ -32,8 +34,9 @@ var log = logf.Log.WithName("metrics") // PrometheusPortName defines the port name used in kubernetes deployment and service resources const PrometheusPortName = "metrics" -// ExposeMetricsPort generates a Kubernetes Service to expose the metrics port -func ExposeMetricsPort(address string) (*v1.Service, error) { +// ExposeMetricsPort creates a Kubernetes Service to expose the metrics port which is extracted, +// from the address passed. +func ExposeMetricsPort(address string, mgr manager.Manager) (*v1.Service, error) { // Split out port from address, to pass to Service object. // We do not need to check the validity of the port, as controller-runtime // would error out and we would never get to this stage. @@ -45,11 +48,21 @@ func ExposeMetricsPort(address string) (*v1.Service, error) { if err != nil { return nil, fmt.Errorf("failed to parse metrics address %s: %v", address, err) } - service, err := initOperatorService(int32(port64), PrometheusPortName) + s, err := initOperatorService(int32(port64), PrometheusPortName) if err != nil { return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } - return service, nil + err = createService(mgr, s) + + return s, nil +} + +func createService(mgr manager.Manager, s *v1.Service) error { + client := mgr.GetClient() + if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil } // initOperatorService returns the static service which exposes specifed port. diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index ba4bca84d31..0d278cacbf2 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -51,7 +51,6 @@ import ( "github.com/operator-framework/operator-sdk/pkg/metrics" "github.com/operator-framework/operator-sdk/pkg/ready" sdkVersion "github.com/operator-framework/operator-sdk/version" - apierrors "k8s.io/apimachinery/pkg/api/errors" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -125,7 +124,11 @@ func main() { os.Exit(1) } - setupMetrics(mgr) + // Create Service object to expose metrics port. + s, err := metrics.ExposeMetricsPort(metricsAddress, mgr) + if err != nil { + log.Info(err) + } log.Info("Starting the Cmd.") @@ -135,18 +138,4 @@ func main() { os.Exit(1) } } - -func setupMetrics(mgr manager.Manager) { - // Expose metrics by creating a Service object - s, err := metrics.ExposeMetricsPort(metricsAddress) - if err != nil { - log.Info(err.Error()) - return - } - client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { - log.Info(err.Error()) - return - } -} ` diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 72ca1a47607..d7136d30b96 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -49,7 +49,6 @@ import ( "github.com/operator-framework/operator-sdk/pkg/metrics" "github.com/operator-framework/operator-sdk/pkg/ready" sdkVersion "github.com/operator-framework/operator-sdk/version" - apierrors "k8s.io/apimachinery/pkg/api/errors" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -123,7 +122,11 @@ func main() { os.Exit(1) } - setupMetrics(mgr) + // Create Service object to expose metrics port. + s, err := metrics.ExposeMetricsPort(metricsAddress, mgr) + if err != nil { + log.Info(err) + } log.Info("Starting the Cmd.") @@ -133,18 +136,4 @@ func main() { os.Exit(1) } } - -func setupMetrics(mgr manager.Manager) { - // Expose metrics by creating a Service object - s, err := metrics.ExposeMetricsPort(metricsAddress) - if err != nil { - log.Info(err.Error()) - return - } - client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { - log.Info(err.Error()) - return - } -} ` From 89144da2bc271905db5c48b6354427a35acbce08 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 21 Jan 2019 15:24:40 +0100 Subject: [PATCH 23/34] pkg/metrics: Add missing import --- pkg/metrics/metrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 630aa10176d..6ad6ec9a397 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -23,6 +23,7 @@ import ( "github.com/operator-framework/operator-sdk/pkg/k8sutil" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/manager" From 8397c393ab1ecedb0870a47317c46f099a770db4 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 21 Jan 2019 15:51:23 +0100 Subject: [PATCH 24/34] pkg/scaffold: Stringify error --- pkg/scaffold/cmd.go | 2 +- pkg/scaffold/cmd_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 0479c7f0eac..ac82989a84e 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -122,7 +122,7 @@ func main() { // Create Service object to expose metrics port. s, err := metrics.ExposeMetricsPort(metricsAddress, mgr) if err != nil { - log.Info(err) + log.Info(err.Error()) } log.Info("Starting the Cmd.") diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 2836f2473ae..2d7171c295f 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -120,7 +120,7 @@ func main() { // Create Service object to expose metrics port. s, err := metrics.ExposeMetricsPort(metricsAddress, mgr) if err != nil { - log.Info(err) + log.Info(err.Error()) } log.Info("Starting the Cmd.") From 23c7acacff882f859bb19a097c924d71b563eead Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 21 Jan 2019 16:23:56 +0100 Subject: [PATCH 25/34] pkg/scaffold: Do not set service --- pkg/scaffold/cmd.go | 2 +- pkg/scaffold/cmd_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index ac82989a84e..2b2e22568b8 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -120,7 +120,7 @@ func main() { } // Create Service object to expose metrics port. - s, err := metrics.ExposeMetricsPort(metricsAddress, mgr) + _, err = metrics.ExposeMetricsPort(metricsAddress, mgr) if err != nil { log.Info(err.Error()) } diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 2d7171c295f..189117da686 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -118,7 +118,7 @@ func main() { } // Create Service object to expose metrics port. - s, err := metrics.ExposeMetricsPort(metricsAddress, mgr) + _, err = metrics.ExposeMetricsPort(metricsAddress, mgr) if err != nil { log.Info(err.Error()) } From c4e1922adb0a06c2712a40d1051c41f99ac649b7 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 21 Jan 2019 18:40:10 +0100 Subject: [PATCH 26/34] pkg*: Accept port32 in func --- pkg/metrics/metrics.go | 18 +++--------------- pkg/scaffold/cmd.go | 16 +++++++++++----- pkg/scaffold/cmd_test.go | 16 +++++++++++----- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 6ad6ec9a397..870c4dc0c17 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -17,8 +17,6 @@ package metrics import ( "context" "fmt" - "net" - "strconv" "github.com/operator-framework/operator-sdk/pkg/k8sutil" @@ -35,21 +33,11 @@ var log = logf.Log.WithName("metrics") // PrometheusPortName defines the port name used in kubernetes deployment and service resources const PrometheusPortName = "metrics" -// ExposeMetricsPort creates a Kubernetes Service to expose the metrics port which is extracted, -// from the address passed. -func ExposeMetricsPort(address string, mgr manager.Manager) (*v1.Service, error) { - // Split out port from address, to pass to Service object. +// ExposeMetricsPort creates a Kubernetes Service to expose the passed metrics port. +func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { // We do not need to check the validity of the port, as controller-runtime // would error out and we would never get to this stage. - _, port, err := net.SplitHostPort(address) - if err != nil { - return nil, fmt.Errorf("failed to split metrics address %s: %v", address, err) - } - port64, err := strconv.ParseInt(port, 0, 32) - if err != nil { - return nil, fmt.Errorf("failed to parse metrics address %s: %v", address, err) - } - s, err := initOperatorService(int32(port64), PrometheusPortName) + s, err := initOperatorService(port, PrometheusPortName) if err != nil { return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index 2b2e22568b8..cc2b1a2cdcf 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -57,8 +57,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) -// Change metricsAddress to serve metrics on different host and port. -var metricsAddress string = ":8383" +// Change below variables to serve metrics on different host or port. +var ( + metricsHost = "0.0.0.0" + metricsPort int32 = 8383 +) var log = logf.Log.WithName("cmd") func printVersion() { @@ -99,7 +102,10 @@ func main() { } // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{Namespace: namespace, MetricsBindAddress: metricsAddress}) + mgr, err := manager.New(cfg, manager.Options{ + Namespace: namespace, + MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), + }) if err != nil { log.Error(err, "") os.Exit(1) @@ -119,8 +125,8 @@ func main() { os.Exit(1) } - // Create Service object to expose metrics port. - _, err = metrics.ExposeMetricsPort(metricsAddress, mgr) + // Create Service object to expose the metrics port. + _, err = metrics.ExposeMetricsPort(metricsPort, mgr) if err != nil { log.Info(err.Error()) } diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index 189117da686..aa43f782387 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -55,8 +55,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) -// Change metricsAddress to serve metrics on different host and port. -var metricsAddress string = ":8383" +// Change below variables to serve metrics on different host or port. +var ( + metricsHost = "0.0.0.0" + metricsPort int32 = 8383 +) var log = logf.Log.WithName("cmd") func printVersion() { @@ -97,7 +100,10 @@ func main() { } // Create a new Cmd to provide shared dependencies and start components - mgr, err := manager.New(cfg, manager.Options{Namespace: namespace, MetricsBindAddress: metricsAddress}) + mgr, err := manager.New(cfg, manager.Options{ + Namespace: namespace, + MetricsBindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort), + }) if err != nil { log.Error(err, "") os.Exit(1) @@ -117,8 +123,8 @@ func main() { os.Exit(1) } - // Create Service object to expose metrics port. - _, err = metrics.ExposeMetricsPort(metricsAddress, mgr) + // Create Service object to expose the metrics port. + _, err = metrics.ExposeMetricsPort(metricsPort, mgr) if err != nil { log.Info(err.Error()) } From e6fe5cc30d31fb38c0764b8f09354642d5f028e1 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 21 Jan 2019 18:59:12 +0100 Subject: [PATCH 27/34] pkg/metrics: Add error handling --- pkg/metrics/metrics.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 870c4dc0c17..8f1c3df23de 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -42,6 +42,9 @@ func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } err = createService(mgr, s) + if err != nil { + return nil, err + } return s, nil } From f2f2e916cc7a35516b77abcc0e8a2e938ed0bc91 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 22 Jan 2019 12:19:34 +0100 Subject: [PATCH 28/34] pkg/metrics: Return Service if it exists --- pkg/metrics/metrics.go | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 8f1c3df23de..0bd41c5df83 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -23,6 +23,7 @@ import ( v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/manager" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" @@ -41,20 +42,36 @@ func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { if err != nil { return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } - err = createService(mgr, s) + service, err := createService(mgr, s) if err != nil { return nil, err } - return s, nil + return service, nil } -func createService(mgr manager.Manager, s *v1.Service) error { +func createService(mgr manager.Manager, s *v1.Service) (*v1.Service, error) { client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil && !apierrors.IsAlreadyExists(err) { - return err + if err := client.Create(context.TODO(), s); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + // Get existing Service and return it + existingService := &v1.Service{} + err := client.Get(context.TODO(), types.NamespacedName{ + Name: s.ObjectMeta.Name, + Namespace: s.ObjectMeta.Namespace, + }, existingService) + if err != nil { + return nil, err + } + log.Info("Metrics Service object already existed") + return existingService, nil + } - return nil + + log.Info("Metrics Service object created") + return s, nil } // initOperatorService returns the static service which exposes specifed port. From 91f8a59756783c54fd38a61a349c31120bd00fe1 Mon Sep 17 00:00:00 2001 From: Haseeb Tariq Date: Thu, 24 Jan 2019 09:44:52 +0100 Subject: [PATCH 29/34] Update pkg/metrics/metrics.go Co-Authored-By: LiliC --- pkg/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 0bd41c5df83..8b8673df7f2 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -70,7 +70,7 @@ func createService(mgr manager.Manager, s *v1.Service) (*v1.Service, error) { } - log.Info("Metrics Service object created") + log.Info("Metrics Service object created", "name", s.Name) return s, nil } From 410be12c31884d0a5a3228dff3561bafe3d471e4 Mon Sep 17 00:00:00 2001 From: Haseeb Tariq Date: Thu, 24 Jan 2019 09:45:10 +0100 Subject: [PATCH 30/34] Update pkg/metrics/metrics.go Co-Authored-By: LiliC --- pkg/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 8b8673df7f2..eb47ea4bee2 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -44,7 +44,7 @@ func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { } service, err := createService(mgr, s) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create or get service for metrics: %v", err) } return service, nil From ddda0f474dc533a39a19bd9bcad0cc12ea71283b Mon Sep 17 00:00:00 2001 From: Haseeb Tariq Date: Thu, 24 Jan 2019 09:45:19 +0100 Subject: [PATCH 31/34] Update pkg/metrics/metrics.go Co-Authored-By: LiliC --- pkg/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index eb47ea4bee2..ce82ae2e616 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -65,7 +65,7 @@ func createService(mgr manager.Manager, s *v1.Service) (*v1.Service, error) { if err != nil { return nil, err } - log.Info("Metrics Service object already existed") + log.Info("Metrics Service object already exists", "name", existingService.Name) return existingService, nil } From 68452c3e47274a18ac764fcf3e20a1ffad320147 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Jan 2019 14:02:17 +0100 Subject: [PATCH 32/34] pkg/metrics/metrics.go: Skip errors when running locally. --- pkg/metrics/metrics.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index ce82ae2e616..d4b80f5de2c 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -40,6 +40,10 @@ func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { // would error out and we would never get to this stage. s, err := initOperatorService(port, PrometheusPortName) if err != nil { + if err == k8sutil.ErrNoNamespace { + log.Info("Skipping metrics Service creation; not running in a cluster.") + return nil, nil + } return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } service, err := createService(mgr, s) From 84d2432caf028abebdcb21f7a1b67f7315a66e56 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Jan 2019 16:21:15 +0100 Subject: [PATCH 33/34] pkg/*: Create new client for Service get/create Due to the cache not being started at the time when we attempt to query for the Service, we instead create a new client in a similiar way as we do with leader elections. --- pkg/metrics/metrics.go | 29 +++++++++++++++++++---------- pkg/scaffold/cmd.go | 6 ++++-- pkg/scaffold/cmd_test.go | 6 ++++-- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index d4b80f5de2c..1ea736ef580 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -25,7 +25,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" - "sigs.k8s.io/controller-runtime/pkg/manager" + "k8s.io/client-go/rest" + crclient "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" ) @@ -35,7 +36,7 @@ var log = logf.Log.WithName("metrics") const PrometheusPortName = "metrics" // ExposeMetricsPort creates a Kubernetes Service to expose the passed metrics port. -func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { +func ExposeMetricsPort(ctx context.Context, port int32) (*v1.Service, error) { // We do not need to check the validity of the port, as controller-runtime // would error out and we would never get to this stage. s, err := initOperatorService(port, PrometheusPortName) @@ -46,7 +47,7 @@ func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { } return nil, fmt.Errorf("failed to initialize service object for metrics: %v", err) } - service, err := createService(mgr, s) + service, err := createService(ctx, s) if err != nil { return nil, fmt.Errorf("failed to create or get service for metrics: %v", err) } @@ -54,24 +55,32 @@ func ExposeMetricsPort(port int32, mgr manager.Manager) (*v1.Service, error) { return service, nil } -func createService(mgr manager.Manager, s *v1.Service) (*v1.Service, error) { - client := mgr.GetClient() - if err := client.Create(context.TODO(), s); err != nil { +func createService(ctx context.Context, s *v1.Service) (*v1.Service, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, err + } + + client, err := crclient.New(config, crclient.Options{}) + if err != nil { + return nil, err + } + + if err := client.Create(ctx, s); err != nil { if !apierrors.IsAlreadyExists(err) { return nil, err } // Get existing Service and return it existingService := &v1.Service{} - err := client.Get(context.TODO(), types.NamespacedName{ - Name: s.ObjectMeta.Name, - Namespace: s.ObjectMeta.Namespace, + err := client.Get(ctx, types.NamespacedName{ + Name: s.Name, + Namespace: s.Namespace, }, existingService) if err != nil { return nil, err } log.Info("Metrics Service object already exists", "name", existingService.Name) return existingService, nil - } log.Info("Metrics Service object created", "name", s.Name) diff --git a/pkg/scaffold/cmd.go b/pkg/scaffold/cmd.go index cc2b1a2cdcf..15aa2f80754 100644 --- a/pkg/scaffold/cmd.go +++ b/pkg/scaffold/cmd.go @@ -93,9 +93,11 @@ func main() { log.Error(err, "") os.Exit(1) } + + ctx := context.TODO() // Become the leader before proceeding - err = leader.Become(context.TODO(), "{{ .ProjectName }}-lock") + err = leader.Become(ctx, "{{ .ProjectName }}-lock") if err != nil { log.Error(err, "") os.Exit(1) @@ -126,7 +128,7 @@ func main() { } // Create Service object to expose the metrics port. - _, err = metrics.ExposeMetricsPort(metricsPort, mgr) + _, err = metrics.ExposeMetricsPort(ctx, metricsPort) if err != nil { log.Info(err.Error()) } diff --git a/pkg/scaffold/cmd_test.go b/pkg/scaffold/cmd_test.go index aa43f782387..62201b83a66 100644 --- a/pkg/scaffold/cmd_test.go +++ b/pkg/scaffold/cmd_test.go @@ -92,8 +92,10 @@ func main() { os.Exit(1) } + ctx := context.TODO() + // Become the leader before proceeding - err = leader.Become(context.TODO(), "app-operator-lock") + err = leader.Become(ctx, "app-operator-lock") if err != nil { log.Error(err, "") os.Exit(1) @@ -124,7 +126,7 @@ func main() { } // Create Service object to expose the metrics port. - _, err = metrics.ExposeMetricsPort(metricsPort, mgr) + _, err = metrics.ExposeMetricsPort(ctx, metricsPort) if err != nil { log.Info(err.Error()) } From c6bf9eadbef307919611f88d2b1d794007124b2a Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Jan 2019 16:45:25 +0100 Subject: [PATCH 34/34] CHANGELOG: Document metrics addition --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f00f5d5b190..5165ccfd797 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### Added +- By default the controller-runtime metrics are exposed on port 8383. This is done as part of the scaffold in the main.go file, the port can be adjusted by modifying the `metricsPort` variable. [#786](https://github.com/operator-framework/operator-sdk/pull/786) + ### Changed ### Deprecated