From 38bbec073cc99b2cca96a4faa3de384463960fee Mon Sep 17 00:00:00 2001 From: Denis Moiseev Date: Thu, 8 Jul 2021 17:01:15 +0200 Subject: [PATCH] extract cloud-config-sync-controller to separate binary, remove redundant rbac permissions --- Dockerfile | 1 + Makefile | 7 +- cmd/cloud-config-sync-controller/main.go | 151 ++++++++++++++++++ .../main.go | 18 +-- ...ler-manager-operator_02_rbac_operator.yaml | 16 -- ...roller-manager-operator_11_deployment.yaml | 25 +++ pkg/controllers/common_consts.go | 2 + 7 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 cmd/cloud-config-sync-controller/main.go diff --git a/Dockerfile b/Dockerfile index 9e8e29efd..5879af136 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ RUN make build FROM registry.ci.openshift.org/ocp/4.9:base COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cluster-controller-manager-operator . +COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/cloud-config-sync-controller . COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/bin/render . COPY --from=builder /go/src/github.com/openshift/cluster-cloud-controller-manager-operator/manifests manifests diff --git a/Makefile b/Makefile index a202d4e5e..68f331ef3 100644 --- a/Makefile +++ b/Makefile @@ -21,12 +21,15 @@ test: generate verify manifests unit unit: hack/unit-tests.sh -# Build operator binary -build: verify operator render +# Build operator binaries +build: operator render cloud-config-sync-controller operator: go build -o bin/cluster-controller-manager-operator cmd/cluster-cloud-controller-manager-operator/main.go +cloud-config-sync-controller: + go build -o bin/cloud-config-sync-controller cmd/cloud-config-sync-controller/main.go + render: go build -o bin/render cmd/render/main.go diff --git a/cmd/cloud-config-sync-controller/main.go b/cmd/cloud-config-sync-controller/main.go new file mode 100644 index 000000000..b9b164b11 --- /dev/null +++ b/cmd/cloud-config-sync-controller/main.go @@ -0,0 +1,151 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "time" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/klog/klogr" + "k8s.io/klog/v2" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/cluster-cloud-controller-manager-operator/pkg/controllers" + // +kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + + // The default durations for the leader electrion operations. + leaseDuration = 120 * time.Second + renewDealine = 110 * time.Second + retryPeriod = 90 * time.Second +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(configv1.AddToScheme(scheme)) + + // +kubebuilder:scaffold:scheme +} + +func main() { + klog.InitFlags(nil) + + metricsAddr := flag.String( + "metrics-bind-address", + ":8080", + "Address for hosting metrics", + ) + + healthAddr := flag.String( + "health-addr", + ":9440", + "The address for health checking.", + ) + + leaderElectResourceNamespace := flag.String( + "leader-elect-resource-namespace", + "", + "The namespace of resource object that is used for locking during leader election. If unspecified and running in cluster, defaults to the service account namespace for the controller. Required for leader-election outside of a cluster.", + ) + + leaderElect := flag.Bool( + "leader-elect", + false, + "Start a leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.", + ) + + leaderElectLeaseDuration := flag.Duration( + "leader-elect-lease-duration", + leaseDuration, + "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.", + ) + + managedNamespace := flag.String( + "namespace", + controllers.DefaultManagedNamespace, + "The namespace for managed objects, target cloud-conf in particular.", + ) + + flag.Parse() + + ctrl.SetLogger(klogr.New().WithName("CCMOCloudConfigSyncController")) + + syncPeriod := 10 * time.Minute + cacheBuilder := cache.MultiNamespacedCacheBuilder([]string{ + *managedNamespace, controllers.OpenshiftConfigNamespace, controllers.OpenshiftManagedConfigNamespace, + }) + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Namespace: *managedNamespace, + Scheme: scheme, + SyncPeriod: &syncPeriod, + MetricsBindAddress: *metricsAddr, + HealthProbeBindAddress: *healthAddr, + LeaderElectionNamespace: *leaderElectResourceNamespace, + LeaderElection: *leaderElect, + LeaseDuration: leaderElectLeaseDuration, + LeaderElectionID: "cloud-config-sync-controller-leader", + RetryPeriod: &retryPeriod, + RenewDeadline: &renewDealine, + NewCache: cacheBuilder, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if err = (&controllers.CloudConfigReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("cloud-controller-manager-operator-config-sync-controller"), + TargetNamespace: *managedNamespace, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create cloud-config sync controller", "controller", "ClusterOperator") + os.Exit(1) + } + // +kubebuilder:scaffold:builder + + if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/cmd/cluster-cloud-controller-manager-operator/main.go b/cmd/cluster-cloud-controller-manager-operator/main.go index e368ae02b..5ef8a99a3 100644 --- a/cmd/cluster-cloud-controller-manager-operator/main.go +++ b/cmd/cluster-cloud-controller-manager-operator/main.go @@ -32,7 +32,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" configv1 "github.com/openshift/api/config/v1" @@ -51,7 +50,6 @@ var ( ) const ( - defaultManagedNamespace = "openshift-cloud-controller-manager" defaultImagesLocation = "/etc/cloud-controller-manager-config/images.json" releaseVersionEnvVariableName = "RELEASE_VERSION" unknownVersionValue = "unknown" @@ -100,7 +98,7 @@ func main() { managedNamespace := flag.String( "namespace", - defaultManagedNamespace, + controllers.DefaultManagedNamespace, "The namespace for managed objects, where out-of-tree CCM binaries will run.", ) @@ -115,9 +113,6 @@ func main() { ctrl.SetLogger(klogr.New().WithName("CCMOperator")) syncPeriod := 10 * time.Minute - cacheBuilder := cache.MultiNamespacedCacheBuilder([]string{ - *managedNamespace, controllers.OpenshiftConfigNamespace, controllers.OpenshiftManagedConfigNamespace, - }) mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Namespace: *managedNamespace, Scheme: scheme, @@ -131,7 +126,6 @@ func main() { LeaderElectionID: "cluster-cloud-controller-manager-leader", RetryPeriod: &retryPeriod, RenewDeadline: &renewDealine, - NewCache: cacheBuilder, }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -149,16 +143,6 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "ClusterOperator") os.Exit(1) } - - if err = (&controllers.CloudConfigReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("cloud-controller-manager-operator"), - TargetNamespace: *managedNamespace, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create cloud-config sync controller", "controller", "ClusterOperator") - os.Exit(1) - } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil { diff --git a/manifests/0000_26_cloud-controller-manager-operator_02_rbac_operator.yaml b/manifests/0000_26_cloud-controller-manager-operator_02_rbac_operator.yaml index 67cec7aa3..434e05568 100644 --- a/manifests/0000_26_cloud-controller-manager-operator_02_rbac_operator.yaml +++ b/manifests/0000_26_cloud-controller-manager-operator_02_rbac_operator.yaml @@ -107,14 +107,6 @@ rules: - get - list - watch - - apiGroups: - - apps - resources: - - deployments - verbs: - - get - - list - - watch --- apiVersion: rbac.authorization.k8s.io/v1 @@ -152,14 +144,6 @@ rules: - get - list - watch - - apiGroups: - - apps - resources: - - deployments - verbs: - - get - - list - - watch --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/manifests/0000_26_cloud-controller-manager-operator_11_deployment.yaml b/manifests/0000_26_cloud-controller-manager-operator_11_deployment.yaml index b3475b505..2f9eb9165 100644 --- a/manifests/0000_26_cloud-controller-manager-operator_11_deployment.yaml +++ b/manifests/0000_26_cloud-controller-manager-operator_11_deployment.yaml @@ -52,6 +52,31 @@ spec: - mountPath: /etc/kubernetes name: host-etc-kube readOnly: true + - name: cloud-config-sync-controller + image: quay.io/openshift/origin-cluster-cloud-controller-manager-operator + command: + - /bin/bash + - -c + - | + #!/bin/bash + set -o allexport + if [[ -f /etc/kubernetes/apiserver-url.env ]]; then + source /etc/kubernetes/apiserver-url.env + else + URL_ONLY_KUBECONFIG=/etc/kubernetes/kubeconfig + fi + exec /cloud-config-sync-controller \ + --leader-elect \ + --metrics-bind-address=:8081 \ + --health-addr=:9441 + resources: + requests: + cpu: 10m + memory: 25Mi + volumeMounts: + - mountPath: /etc/kubernetes + name: host-etc-kube + readOnly: true hostNetwork: true nodeSelector: node-role.kubernetes.io/master: "" diff --git a/pkg/controllers/common_consts.go b/pkg/controllers/common_consts.go index 5566d5b41..18947fa0a 100644 --- a/pkg/controllers/common_consts.go +++ b/pkg/controllers/common_consts.go @@ -1,6 +1,8 @@ package controllers const ( + DefaultManagedNamespace = "openshift-cloud-controller-manager" + infrastructureResourceName = "cluster" OpenshiftConfigNamespace = "openshift-config"