MCO-552: implement the ability for the MCO to handle image registry certificates

cmd/machine-config-operator/start.go

@@ -92,6 +92,7 @@ func runStartCmd(_ *cobra.Command, _ []string) {
 			ctrlctx.OpenShiftKubeAPIServerKubeNamespacedInformerFactory.Core().V1().ConfigMaps(),
 			ctrlctx.KubeInformerFactory.Core().V1().Nodes(),
 			ctrlctx.KubeMAOSharedInformer.Core().V1().Secrets(),
+			ctrlctx.ConfigInformerFactory.Config().V1().Images(),
 		)
 
 		ctrlctx.NamespacedInformerFactory.Start(ctrlctx.Stop)

lib/resourcemerge/machineconfig.go

@@ -82,6 +82,15 @@ func ensureControllerConfigSpec(modified *bool, existing *mcfgv1.ControllerConfi
 	setBytesIfSet(modified, &existing.KubeAPIServerServingCAData, required.KubeAPIServerServingCAData)
 	setBytesIfSet(modified, &existing.CloudProviderCAData, required.CloudProviderCAData)
 
+	if required.ImageRegistryBundleData != nil && !equality.Semantic.DeepEqual(existing.ImageRegistryBundleData, required.ImageRegistryBundleData) {
+		*modified = true
+		existing.ImageRegistryBundleData = required.ImageRegistryBundleData
+	}
+
+	if required.ImageRegistryBundleUserData != nil && !equality.Semantic.DeepEqual(existing.ImageRegistryBundleUserData, required.ImageRegistryBundleUserData) {
+		*modified = true
+		existing.ImageRegistryBundleUserData = required.ImageRegistryBundleUserData
+	}
 	if required.Infra != nil && !equality.Semantic.DeepEqual(existing.Infra, required.Infra) {
 		*modified = true
 		existing.Infra = required.Infra

manifests/controllerconfig.crd.yaml

@@ -170,6 +170,34 @@ spec:
                 description: images is map of images that are used by the controller
                   to render templates under ./templates/
                 type: object
+              imageRegistryBundleUserData:
+                description: imageRegistryBundleUserData is Image Registry Data provided by the user. This is pulled from image.config.openshift.io/cluster which contains an additionalCA field.
+                items:
+                  description: ImageRegistryBundle contains the file and data for each CA
+                  properties:
+                    file:
+                      description: file of the CA
+                      type: string
+                    data:
+                      description: data of the CA
+                      type: string
+                      format: byte
+                  type: object
+                type: array
+              imageRegistryBundleData:
+                description: imageRegistryBundleData is Image Registry Data provided by the openshift-config-managed/image-registry-ca configmap.
+                items:
+                  description: ImageRegistryBundle contains the file and data for each CA
+                  properties:
+                    file:
+                      description: file of the CA
+                      type: string
+                    data:
+                      description: data of the CA
+                      type: string
+                      format: byte
+                  type: object
+                type: array
               infra:
                 description: infra holds the infrastructure details
                 nullable: true

pkg/apis/machineconfiguration.openshift.io/v1/types.go

@@ -63,6 +63,12 @@ type ControllerConfigSpec struct {
 	// +nullable
 	AdditionalTrustBundle []byte `json:"additionalTrustBundle"`
 
+	// imageRegistryBundleUserData is Image Registry Data provided by the user
+	ImageRegistryBundleUserData []ImageRegistryBundle `json:"imageRegistryBundleUserData"`
+
+	// imageRegistryBundleData is the ImageRegistryData
+	ImageRegistryBundleData []ImageRegistryBundle `json:"imageRegistryBundleData"`
+
 	// TODO: Investigate using a ConfigMapNameReference for the PullSecret and OSImageURL
 
 	// pullSecret is the default pull secret that needs to be installed
@@ -113,6 +119,11 @@ type ControllerConfigSpec struct {
 	Network *NetworkInfo `json:"network"`
 }
 
+type ImageRegistryBundle struct {
+	File string `json:"file"`
+	Data []byte `json:"data"`
+}
+
 // IPFamiliesType indicates whether the cluster network is IPv4-only, IPv6-only, or dual-stack
 type IPFamiliesType string
 

pkg/controller/template/template_controller.go

@@ -7,6 +7,8 @@ import (
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
+	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"time"
@@ -430,39 +432,86 @@ func updateControllerConfigCerts(config *mcfgv1.ControllerConfig) bool {
 	names := []string{
 		"KubeAPIServerServingCAData", "CloudProviderCAData", "RootCAData", "AdditionalTrustBundle",
 	}
-	config.Status.ControllerCertificates = []v1.ControllerCertificate{}
 	certs := [][]byte{
 		config.Spec.KubeAPIServerServingCAData,
 		config.Spec.CloudProviderCAData,
 		config.Spec.RootCAData,
 		config.Spec.AdditionalTrustBundle,
 	}
+	newImgCerts := []v1.ControllerCertificate{}
+	newCtrlCerts := []v1.ControllerCertificate{}
 	for i, cert := range certs {
-		for len(cert) > 0 {
-			b, next := pem.Decode(cert)
-			if b == nil {
+		certs := createNewCert(cert, names[i])
+		if len(certs) > 0 {
+			modified = true
+			newCtrlCerts = append(newCtrlCerts, certs...)
+		}
+	}
+	for _, entry := range config.Spec.ImageRegistryBundleData {
+		names = append(names, entry.File)
+		certs := createNewCert(entry.Data, entry.File)
+		if len(certs) > 0 {
+			modified = true
+			newImgCerts = append(newImgCerts, certs...)
+		}
+	}
+	for _, entry := range config.Spec.ImageRegistryBundleUserData {
+		names = append(names, entry.File)
+		certs := createNewCert(entry.Data, entry.File)
+		if len(certs) > 0 {
+			modified = true
+			newImgCerts = append(newImgCerts, certs...)
+		}
+	}
+	stillExists := false
+	for _, cert := range config.Status.ControllerCertificates {
+		// skip the non-IR certs
+		if cert.BundleFile == "KubeAPIServerServingCAData" || cert.BundleFile == "CloudProviderCAData" || cert.BundleFile == "RootCAData" || cert.BundleFile == "AdditionalTrustBundle" {
+			continue
+		}
+		for _, newC := range newImgCerts {
+			if newC.BundleFile == cert.BundleFile {
+				stillExists = true
 				break
 			}
-			c, err := x509.ParseCertificate(b.Bytes)
-			if err != nil {
-				klog.Infof("Malformed Cert, not syncing")
-				continue
+		}
+		// need to remove old cert path if it does not still exists (only applies to img certs)
+		if !stillExists {
+			if err := os.RemoveAll(filepath.Join("/etc/docker/certs.d", cert.BundleFile)); err != nil {
+				klog.Infof("Could not remove old certificate: %s", filepath.Join("/etc/docker/certs.d", cert.BundleFile))
 			}
-			cert = next
-			config.Status.ControllerCertificates = append(config.Status.ControllerCertificates, v1.ControllerCertificate{
-				Subject:    c.Subject.String(),
-				Signer:     c.Issuer.String(),
-				NotBefore:  c.NotBefore.String(),
-				NotAfter:   c.NotAfter.String(),
-				BundleFile: names[i],
-			})
-			modified = true
 		}
-
+		stillExists = false
 	}
+	config.Status.ControllerCertificates = append(newCtrlCerts, newImgCerts...)
 	return modified
 }
 
+func createNewCert(cert []byte, name string) []v1.ControllerCertificate {
+	certs := []v1.ControllerCertificate{}
+	for len(cert) > 0 {
+		b, next := pem.Decode(cert)
+		if b == nil {
+			klog.Infof("Unable to decode cert %s into a pem block. Cert is either empty or invalid.", string(cert))
+			break
+		}
+		c, err := x509.ParseCertificate(b.Bytes)
+		if err != nil {
+			klog.Infof("Malformed Cert, not syncing")
+			continue
+		}
+		cert = next
+		certs = append(certs, v1.ControllerCertificate{
+			Subject:    c.Subject.String(),
+			Signer:     c.Issuer.String(),
+			NotBefore:  c.NotBefore.String(),
+			NotAfter:   c.NotAfter.String(),
+			BundleFile: name,
+		})
+	}
+	return certs
+}
+
 // syncControllerConfig will sync the controller config with the given key.
 // This function is not meant to be invoked concurrently with the same key.
 func (ctrl *Controller) syncControllerConfig(key string) error {

pkg/daemon/certificate_writer.go

@@ -2,6 +2,8 @@ package daemon
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 
 	"time"
 
@@ -109,6 +111,24 @@ func (dn *Daemon) syncControllerConfigHandler(key string) error {
 			return err
 		}
 
+		for _, CA := range controllerConfig.Spec.ImageRegistryBundleData {
+			if err := os.MkdirAll(filepath.Join(imageCAFilePath, CA.File), defaultDirectoryPermissions); err != nil {
+				return err
+			}
+			if err := writeFileAtomicallyWithDefaults(filepath.Join(imageCAFilePath, CA.File, "ca.crt"), CA.Data); err != nil {
+				return err
+			}
+		}
+
+		for _, CA := range controllerConfig.Spec.ImageRegistryBundleUserData {
+			if err := os.MkdirAll(filepath.Join(imageCAFilePath, CA.File), defaultDirectoryPermissions); err != nil {
+				return err
+			}
+			if err := writeFileAtomicallyWithDefaults(filepath.Join(imageCAFilePath, CA.File, "ca.crt"), CA.Data); err != nil {
+				return err
+			}
+		}
+
 		annos := map[string]string{
 			constants.ControllerConfigResourceVersionKey: controllerConfig.ObjectMeta.ResourceVersion,
 		}

pkg/daemon/daemon.go

@@ -176,6 +176,8 @@ const (
 	configMapConfigKey          = "config"
 	configMapHashKey            = "hash"
 
+	imageCAFilePath = "/etc/docker/certs.d"
+
 	// used for certificate syncing
 	caBundleFilePath = "/etc/kubernetes/kubelet-ca.crt"
 

pkg/operator/operator.go

@@ -72,6 +72,7 @@ type Operator struct {
 
 	syncHandler func(ic string) error
 
+	imgLister        configlistersv1.ImageLister
 	crdLister        apiextlistersv1.CustomResourceDefinitionLister
 	mcpLister        mcfglistersv1.MachineConfigPoolLister
 	ccLister         mcfglistersv1.ControllerConfigLister
@@ -105,6 +106,7 @@ type Operator struct {
 	nodeListerSynced                 cache.InformerSynced
 	dnsListerSynced                  cache.InformerSynced
 	maoSecretInformerSynced          cache.InformerSynced
+	imgListerSynced                  cache.InformerSynced
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
 	queue workqueue.RateLimitingInterface
@@ -139,6 +141,7 @@ func New(
 	oseKubeAPIInformer coreinformersv1.ConfigMapInformer,
 	nodeInformer coreinformersv1.NodeInformer,
 	maoSecretInformer coreinformersv1.SecretInformer,
+	imgInformer configinformersv1.ImageInformer,
 ) *Operator {
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(klog.Infof)
@@ -172,6 +175,7 @@ func New(
 		clusterRoleInformer.Informer(),
 		clusterRoleBindingInformer.Informer(),
 		mcoCmInformer.Informer(),
+		clusterCmInfomer.Informer(),
 		infraInformer.Informer(),
 		networkInformer.Informer(),
 		mcpInformer.Informer(),
@@ -186,6 +190,7 @@ func New(
 
 	optr.syncHandler = optr.sync
 
+	optr.imgLister = imgInformer.Lister()
 	optr.clusterCmLister = clusterCmInfomer.Lister()
 	optr.clusterCmListerSynced = clusterCmInfomer.Informer().HasSynced
 	optr.mcpLister = mcpInformer.Lister()
@@ -201,6 +206,7 @@ func New(
 	optr.nodeLister = nodeInformer.Lister()
 	optr.nodeListerSynced = nodeInformer.Informer().HasSynced
 
+	optr.imgListerSynced = imgInformer.Informer().HasSynced
 	optr.maoSecretInformerSynced = maoSecretInformer.Informer().HasSynced
 	optr.serviceAccountInformerSynced = serviceAccountInfomer.Informer().HasSynced
 	optr.clusterRoleInformerSynced = clusterRoleInformer.Informer().HasSynced