diff --git a/knative-operator/deploy/resources/monitoring/role-service-monitor.yaml b/knative-operator/deploy/resources/monitoring/role-service-monitor.yaml deleted file mode 100644 index c3d4c0376b..0000000000 --- a/knative-operator/deploy/resources/monitoring/role-service-monitor.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: knative-serving-prometheus-k8s -rules: - - apiGroups: - - "" - resources: - - services - - endpoints - - pods - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: knative-serving-prometheus-k8s -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: knative-serving-prometheus-k8s -subjects: - - kind: ServiceAccount - name: prometheus-k8s - namespace: openshift-monitoring diff --git a/knative-operator/deploy/resources/monitoring/source-service-monitor.yaml b/knative-operator/deploy/resources/monitoring/source-service-monitor.yaml deleted file mode 100644 index 2c936264d7..0000000000 --- a/knative-operator/deploy/resources/monitoring/source-service-monitor.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -spec: - endpoints: - - port: http-metrics - namespaceSelector: {} - diff --git a/knative-operator/deploy/resources/monitoring/source-service.yaml b/knative-operator/deploy/resources/monitoring/source-service.yaml deleted file mode 100644 index 51f4a5f04c..0000000000 --- a/knative-operator/deploy/resources/monitoring/source-service.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -spec: - ports: - - name: http-metrics - port: 9090 - protocol: TCP - targetPort: 9090 - sessionAffinity: None - type: ClusterIP diff --git a/knative-operator/pkg/common/monitoring.go b/knative-operator/pkg/common/monitoring.go index 24c354d93c..9eca51131f 100644 --- a/knative-operator/pkg/common/monitoring.go +++ b/knative-operator/pkg/common/monitoring.go @@ -10,11 +10,13 @@ import ( monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -23,8 +25,6 @@ const ( operatorDeploymentNameEnvKey = "DEPLOYMENT_NAME" // service monitor created successfully when monitoringLabel added to namespace monitoringLabel = "openshift.io/cluster-monitoring" - rolePath = "deploy/resources/monitoring/role-service-monitor.yaml" - TestRolePath = "TEST_ROLE_PATH" ) func SetupMonitoringRequirements(api client.Client, instance mf.Owner) error { @@ -32,7 +32,7 @@ func SetupMonitoringRequirements(api client.Client, instance mf.Owner) error { if err != nil { return err } - err = createRoleAndRoleBinding(instance, instance.GetNamespace(), getRolePath(), api) + err = createRoleAndRoleBinding(instance, instance.GetNamespace(), api) if err != nil { return err } @@ -94,15 +94,6 @@ func GetServerlessOperatorDeployment(api client.Client, namespace string) (*apps return deployment, nil } -func getRolePath() string { - // meant for testing only - ns, found := os.LookupEnv(TestRolePath) - if found { - return ns - } - return rolePath -} - func addMonitoringLabelToNamespace(namespace string, api client.Client) error { ns := &v1.Namespace{} if err := api.Get(context.TODO(), client.ObjectKey{Name: namespace}, ns); err != nil { @@ -118,21 +109,65 @@ func addMonitoringLabelToNamespace(namespace string, api client.Client) error { return nil } -func createRoleAndRoleBinding(instance mf.Owner, namespace, path string, client client.Client) error { - manifest, err := mf.NewManifest(path, mf.UseClient(mfclient.NewClient(client))) +func createRoleAndRoleBinding(instance mf.Owner, namespace string, client client.Client) error { + clientOptions := mf.UseClient(mfclient.NewClient(client)) + rbacManifest, err := createRBACManifestForPrometheusAccount(namespace, clientOptions) if err != nil { - return fmt.Errorf("unable to create role and roleBinding ServiceMonitor install manifest: %w", err) + return err } - transforms := []mf.Transformer{mf.InjectOwner(instance), injectNameSpace(namespace)} - if manifest, err = manifest.Transform(transforms...); err != nil { - return fmt.Errorf("unable to transform role and roleBinding serviceMonitor manifest: %w", err) + transforms := []mf.Transformer{mf.InjectOwner(instance)} + if *rbacManifest, err = rbacManifest.Transform(transforms...); err != nil { + return fmt.Errorf("unable to transform role and roleBinding manifest for Prometheus account: %w", err) } - if err := manifest.Apply(); err != nil { - return fmt.Errorf("unable to create role and roleBinding for ServiceMonitor %w", err) + if err := rbacManifest.Apply(); err != nil { + return fmt.Errorf("unable to create role and roleBinding for Prometheus account %w", err) } return nil } +func createRBACManifestForPrometheusAccount(ns string, options mf.Option) (*mf.Manifest, error) { + var roleU = &unstructured.Unstructured{} + var rbU = &unstructured.Unstructured{} + role := rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: "knative-serving-prometheus-k8s", + Namespace: ns, + }, + Rules: []rbacv1.PolicyRule{{ + APIGroups: []string{""}, + Resources: []string{"services", "endpoints", "pods"}, + Verbs: []string{"get", "list", "watch"}, + }}, + } + rb := rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "knative-serving-prometheus-k8s", + Namespace: ns, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: role.Name, + }, + Subjects: []rbacv1.Subject{{ + Kind: "ServiceAccount", + Name: "prometheus-k8s", + Namespace: "openshift-monitoring", + }}, + } + if err := scheme.Scheme.Convert(&role, roleU, nil); err != nil { + return nil, err + } + if err := scheme.Scheme.Convert(&rb, rbU, nil); err != nil { + return nil, err + } + rbacManifest, err := mf.ManifestFrom(mf.Slice([]unstructured.Unstructured{*roleU, *rbU}), options) + if err != nil { + return nil, err + } + return &rbacManifest, nil +} + func getOperatorDeploymentName() (string, error) { ns, found := os.LookupEnv(operatorDeploymentNameEnvKey) if !found { @@ -140,15 +175,3 @@ func getOperatorDeploymentName() (string, error) { } return ns, nil } - -// Use a custom transformation otherwise if mf.InjectNameSpace was used -// it would wrongly update rolebinding subresource namespace as well -func injectNameSpace(namespace string) mf.Transformer { - return func(u *unstructured.Unstructured) error { - kind := u.GetKind() - if kind == "Role" || kind == "RoleBinding" { - u.SetNamespace(namespace) - } - return nil - } -} diff --git a/knative-operator/pkg/common/monitoring_test.go b/knative-operator/pkg/common/monitoring_test.go index 8fa9046943..d873447b7e 100644 --- a/knative-operator/pkg/common/monitoring_test.go +++ b/knative-operator/pkg/common/monitoring_test.go @@ -30,7 +30,6 @@ var ( func init() { os.Setenv(operatorDeploymentNameEnvKey, "knative-openshift") - os.Setenv(TestRolePath, "testdata/role-service-monitor.yaml") } func TestSetupMonitoringRequirements(t *testing.T) { diff --git a/knative-operator/pkg/common/service_monitor.go b/knative-operator/pkg/common/service_monitor.go index b7e848b9da..267881b5b2 100644 --- a/knative-operator/pkg/common/service_monitor.go +++ b/knative-operator/pkg/common/service_monitor.go @@ -1,9 +1,7 @@ package common import ( - "context" "fmt" - "os" mfclient "github.com/manifestival/controller-runtime-client" mf "github.com/manifestival/manifestival" @@ -12,97 +10,76 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/scheme" "knative.dev/pkg/kmeta" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( - EventingSourceServiceMonitorPath = "deploy/resources/monitoring/source-service-monitor.yaml" - EventingSourcePath = "deploy/resources/monitoring/source-service.yaml" - SourceLabel = "eventing.knative.dev/source" - SourceNameLabel = "eventing.knative.dev/sourceName" - SourceRoleLabel = "sources.knative.dev/role" - TestMonitor = "TEST_MONITOR" - TestSourceServiceMonitorPath = "TEST_SOURCE_SERVICE_MONITOR_PATH" - TestSourceServicePath = "TEST_SOURCE_SERVICE_PATH" + SourceLabel = "eventing.knative.dev/source" + SourceNameLabel = "eventing.knative.dev/sourceName" + SourceRoleLabel = "sources.knative.dev/role" ) -func SetupSourceServiceMonitor(client client.Client, instance *appsv1.Deployment) error { +func SetupSourceServiceMonitorResources(client client.Client, instance *appsv1.Deployment) error { labels := instance.Spec.Selector.MatchLabels - clientOptions := mf.UseClient(mfclient.NewClient(client)) - // create service for the deployment - manifest, err := mf.NewManifest(getMonitorPath(TestSourceServicePath, EventingSourcePath), clientOptions) + // Create service monitor resources for source + smManifest, err := createServiceMonitorManifest(labels, instance.Name, instance.Namespace, clientOptions) if err != nil { - return fmt.Errorf("unable to parse source service manifest: %w", err) - } - transforms := []mf.Transformer{updateService(labels, instance.Name), mf.InjectOwner(instance), mf.InjectNamespace(instance.Namespace)} - if manifest, err = manifest.Transform(transforms...); err != nil { - return fmt.Errorf("unable to transform source service manifest: %w", err) - } - if err := manifest.Apply(); err != nil { return err } - - // get service back, needed for the UID and setting owner refs - srv := &v1.Service{} - if err := client.Get(context.TODO(), types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, srv); err != nil { - return err - } - // create service monitor for source - manifest, err = mf.NewManifest(getMonitorPath(TestSourceServiceMonitorPath, EventingSourceServiceMonitorPath), clientOptions) - if err != nil { - return fmt.Errorf("unable to parse source service monitor manifest: %w", err) - } - transforms = []mf.Transformer{updateServiceMonitor(labels, instance.Name), mf.InjectOwner(srv), mf.InjectNamespace(instance.Namespace)} - if manifest, err = manifest.Transform(transforms...); err != nil { + if *smManifest, err = smManifest.Transform(mf.InjectOwner(instance)); err != nil { return fmt.Errorf("unable to transform source service monitor manifest: %w", err) } - return manifest.Apply() + return smManifest.Apply() } -func getMonitorPath(envVar string, defaultVal string) string { - path := os.Getenv(envVar) - if path == "" { - return defaultVal +func createServiceMonitorManifest(labels map[string]string, depName string, ns string, options mf.Option) (*mf.Manifest, error) { + var svU = &unstructured.Unstructured{} + var smU = &unstructured.Unstructured{} + sms := v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: depName, + Namespace: ns, + Labels: kmeta.CopyMap(labels), + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{{ + Name: "http-metrics", + Port: 9090, + TargetPort: intstr.FromInt(9090), + Protocol: "TCP", + }}, + Selector: kmeta.CopyMap(labels), + }} + sms.Labels["name"] = sms.Name + if err := scheme.Scheme.Convert(&sms, svU, nil); err != nil { + return nil, err } - return path -} - -func updateService(labels map[string]string, depName string) mf.Transformer { - return func(resource *unstructured.Unstructured) error { - if resource.GetKind() != "Service" { - return nil - } - var svc = &v1.Service{} - if err := scheme.Scheme.Convert(resource, svc, nil); err != nil { - return err - } - svc.Name = depName - svc.Labels = kmeta.CopyMap(labels) - svc.Spec.Selector = kmeta.CopyMap(labels) - svc.Labels["name"] = svc.Name - return scheme.Scheme.Convert(svc, resource, nil) + sm := monitoringv1.ServiceMonitor{ + ObjectMeta: metav1.ObjectMeta{ + Name: depName, + Namespace: ns, + Labels: kmeta.CopyMap(labels), + }, + Spec: monitoringv1.ServiceMonitorSpec{ + Endpoints: []monitoringv1.Endpoint{{Port: "http-metrics"}}, + NamespaceSelector: monitoringv1.NamespaceSelector{ + MatchNames: []string{ns}, + }, + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"name": depName}, + }, + }} + sm.Labels["name"] = sm.Name + if err := scheme.Scheme.Convert(&sm, smU, nil); err != nil { + return nil, err } -} - -func updateServiceMonitor(labels map[string]string, depName string) mf.Transformer { - return func(resource *unstructured.Unstructured) error { - if resource.GetKind() != "ServiceMonitor" { - return nil - } - var sm = &monitoringv1.ServiceMonitor{} - if err := scheme.Scheme.Convert(resource, sm, nil); err != nil { - return err - } - sm.Name = depName - sm.Labels = kmeta.CopyMap(labels) - sm.Spec.Selector = metav1.LabelSelector{ - MatchLabels: map[string]string{"name": sm.Name}, - } - sm.Labels["name"] = sm.Name - return scheme.Scheme.Convert(sm, resource, nil) + smManifest, err := mf.ManifestFrom(mf.Slice([]unstructured.Unstructured{*svU, *smU}), options) + if err != nil { + return nil, err } + return &smManifest, nil } diff --git a/knative-operator/pkg/common/sources/source_deployment_discovery_controller.go b/knative-operator/pkg/common/sources/source_deployment_discovery_controller.go index 4a83a3c55f..062ef01148 100644 --- a/knative-operator/pkg/common/sources/source_deployment_discovery_controller.go +++ b/knative-operator/pkg/common/sources/source_deployment_discovery_controller.go @@ -82,7 +82,7 @@ func (r *ReconcileSourceDeployment) Reconcile(ctx context.Context, request recon if err := common.SetupMonitoringRequirements(r.client, dep); err != nil { return reconcile.Result{}, err } - if err := common.SetupSourceServiceMonitor(r.client, dep); err != nil { + if err := common.SetupSourceServiceMonitorResources(r.client, dep); err != nil { return reconcile.Result{}, err } return reconcile.Result{}, nil diff --git a/knative-operator/pkg/common/sources/source_deployment_discovery_controller_test.go b/knative-operator/pkg/common/sources/source_deployment_discovery_controller_test.go index d3c964b838..6b155540e7 100644 --- a/knative-operator/pkg/common/sources/source_deployment_discovery_controller_test.go +++ b/knative-operator/pkg/common/sources/source_deployment_discovery_controller_test.go @@ -68,10 +68,6 @@ var ( func init() { os.Setenv("OPERATOR_NAME", "TEST_OPERATOR") - os.Setenv(common.TestRolePath, "../testdata/role-service-monitor.yaml") - os.Setenv(common.TestSourceServiceMonitorPath, "../testdata/source-service-monitor.yaml") - os.Setenv(common.TestSourceServicePath, "../testdata/source-service.yaml") - apis.AddToScheme(scheme.Scheme) } diff --git a/knative-operator/pkg/common/testdata/role-service-monitor.yaml b/knative-operator/pkg/common/testdata/role-service-monitor.yaml deleted file mode 100644 index c3d4c0376b..0000000000 --- a/knative-operator/pkg/common/testdata/role-service-monitor.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: knative-serving-prometheus-k8s -rules: - - apiGroups: - - "" - resources: - - services - - endpoints - - pods - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: knative-serving-prometheus-k8s -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: knative-serving-prometheus-k8s -subjects: - - kind: ServiceAccount - name: prometheus-k8s - namespace: openshift-monitoring diff --git a/knative-operator/pkg/common/testdata/source-service-monitor.yaml b/knative-operator/pkg/common/testdata/source-service-monitor.yaml deleted file mode 100644 index 2c936264d7..0000000000 --- a/knative-operator/pkg/common/testdata/source-service-monitor.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -spec: - endpoints: - - port: http-metrics - namespaceSelector: {} - diff --git a/knative-operator/pkg/common/testdata/source-service.yaml b/knative-operator/pkg/common/testdata/source-service.yaml deleted file mode 100644 index 51f4a5f04c..0000000000 --- a/knative-operator/pkg/common/testdata/source-service.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -spec: - ports: - - name: http-metrics - port: 9090 - protocol: TCP - targetPort: 9090 - sessionAffinity: None - type: ClusterIP diff --git a/knative-operator/pkg/controller/knativeeventing/knativeeventing_controller_test.go b/knative-operator/pkg/controller/knativeeventing/knativeeventing_controller_test.go index 8c0af0c0f4..ddb2e99592 100644 --- a/knative-operator/pkg/controller/knativeeventing/knativeeventing_controller_test.go +++ b/knative-operator/pkg/controller/knativeeventing/knativeeventing_controller_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/openshift-knative/serverless-operator/knative-operator/pkg/apis" - "github.com/openshift-knative/serverless-operator/knative-operator/pkg/common" "github.com/openshift-knative/serverless-operator/knative-operator/pkg/controller/dashboard" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -45,8 +44,6 @@ func init() { os.Setenv(dashboard.EventingSourceDashboardPathEnvVar, "../dashboard/testdata/grafana-dash-knative-eventing-source.yaml") os.Setenv(dashboard.EventingBrokerDashboardPathEnvVar, "../dashboard/testdata/grafana-dash-knative-eventing-broker.yaml") os.Setenv(dashboard.EventingResourceDashboardPathEnvVar, "../dashboard/testdata/grafana-dash-knative-eventing-resources.yaml") - os.Setenv(common.TestRolePath, "../dashboard/testdata/role-service-monitor.yaml") - os.Setenv(common.TestMonitor, "true") apis.AddToScheme(scheme.Scheme) } diff --git a/knative-operator/pkg/controller/knativeserving/knativeserving_controller_test.go b/knative-operator/pkg/controller/knativeserving/knativeserving_controller_test.go index ff382b493f..9fcec472cb 100644 --- a/knative-operator/pkg/controller/knativeserving/knativeserving_controller_test.go +++ b/knative-operator/pkg/controller/knativeserving/knativeserving_controller_test.go @@ -125,7 +125,6 @@ func init() { os.Setenv("OPERATOR_NAME", "TEST_OPERATOR") os.Setenv(quickstart.EnvKey, "../../../deploy/resources/quickstart/serverless-application-quickstart.yaml") os.Setenv(dashboard.ServingResourceDashboardPathEnvVar, "../dashboard/testdata/grafana-dash-knative-serving-resources.yaml") - os.Setenv("TEST_ROLE_PATH", "../dashboard/testdata/role-service-monitor.yaml") apis.AddToScheme(scheme.Scheme) }