diff --git a/go.mod b/go.mod index caec03fe4..0fe3553dd 100644 --- a/go.mod +++ b/go.mod @@ -17,13 +17,14 @@ require ( github.com/stretchr/testify v1.7.0 go.uber.org/atomic v1.9.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 k8s.io/api v0.25.2 k8s.io/apiextensions-apiserver v0.24.2 k8s.io/apimachinery v0.25.2 + k8s.io/apiserver v0.24.2 k8s.io/client-go v0.25.2 - k8s.io/component-base v0.24.2 + k8s.io/component-base v0.25.2 k8s.io/klog/v2 v2.70.1 k8s.io/metrics v0.25.2 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed @@ -42,7 +43,7 @@ require ( github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/go-logr/zapr v1.2.0 // indirect + github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/swag v0.21.1 // indirect diff --git a/go.sum b/go.sum index 04a920b29..2d11a2022 100644 --- a/go.sum +++ b/go.sum @@ -165,10 +165,12 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -647,8 +649,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -961,13 +964,17 @@ k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZf k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.25.2 h1:WbxfAjCx+AeN8Ilp9joWnyJ6xu9OMeS/fsfjK/5zaQs= k8s.io/apimachinery v0.25.2/go.mod h1:hqqA1X0bsgsxI6dXsJ4HnNTBOmJNxyPp8dw3u2fSHwA= +k8s.io/apiserver v0.24.2 h1:orxipm5elPJSkkFNlwH9ClqaKEDJJA3yR2cAAlCnyj4= k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= +k8s.io/apiserver v0.25.2 h1:YePimobk187IMIdnmsMxsfIbC5p4eX3WSOrS9x6FEYw= +k8s.io/apiserver v0.25.2/go.mod h1:30r7xyQTREWCkG2uSjgjhQcKVvAAlqoD+YyrqR6Cn+I= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= k8s.io/client-go v0.25.2 h1:SUPp9p5CwM0yXGQrwYurw9LWz+YtMwhWd0GqOsSiefo= k8s.io/client-go v0.25.2/go.mod h1:i7cNU7N+yGQmJkewcRD2+Vuj4iz7b30kI8OcL3horQ4= k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= -k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= +k8s.io/component-base v0.25.2 h1:Nve/ZyHLUBHz1rqwkjXm/Re6IniNa5k7KgzxZpTfSQY= +k8s.io/component-base v0.25.2/go.mod h1:90W21YMr+Yjg7MX+DohmZLzjsBtaxQDDwaX4YxDkl60= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= diff --git a/pkg/controllers/work/apply_controller.go b/pkg/controllers/work/apply_controller.go index 2f033f5e3..7e8ead482 100644 --- a/pkg/controllers/work/apply_controller.go +++ b/pkg/controllers/work/apply_controller.go @@ -20,6 +20,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "time" @@ -32,6 +33,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apiserver/pkg/storage/names" "k8s.io/client-go/dynamic" "k8s.io/client-go/tools/record" "k8s.io/klog/v2" @@ -50,6 +52,8 @@ import ( const ( workFieldManagerName = "work-api-agent" + fleetSystemNamespace = "fleet-system" + configMapNamePrefix = "configmap-" ) // ApplyWorkReconciler reconciles a Work object @@ -298,7 +302,7 @@ func (r *ApplyWorkReconciler) applyManifests(ctx context.Context, manifests []wo default: addOwnerRef(owner, rawObj) - appliedObj, result.action, result.err = r.applyUnstructured(ctx, gvr, rawObj) + appliedObj, result.action, result.err = r.applyUnstructured(ctx, gvr, rawObj, owner) result.identifier = buildResourceIdentifier(index, rawObj, gvr) logObjRef := klog.ObjectRef{ Name: result.identifier.Name, @@ -335,26 +339,24 @@ func (r *ApplyWorkReconciler) decodeManifest(manifest workv1alpha1.Manifest) (sc // Determines if an unstructured manifest object can & should be applied. If so, it applies (creates) the resource on the cluster. func (r *ApplyWorkReconciler) applyUnstructured(ctx context.Context, gvr schema.GroupVersionResource, - manifestObj *unstructured.Unstructured) (*unstructured.Unstructured, applyAction, error) { + manifestObj *unstructured.Unstructured, owner metav1.OwnerReference) (*unstructured.Unstructured, applyAction, error) { manifestRef := klog.ObjectRef{ Name: manifestObj.GetName(), Namespace: manifestObj.GetNamespace(), } - // compute the hash without taking into consider the last applied annotation - if err := setManifestHashAnnotation(manifestObj); err != nil { - return nil, ManifestNoChangeAction, err - } // extract the common create procedure to reuse var createFunc = func() (*unstructured.Unstructured, applyAction, error) { - // record the raw manifest with the hash annotation in the manifest - if err := setModifiedConfigurationAnnotation(manifestObj); err != nil { - return nil, ManifestNoChangeAction, err - } + configMapName := getRandomConfigMapName() + // set config map name annotation on manifest + setConfigMapNameAnnotation(configMapName, manifestObj) actual, err := r.spokeDynamicClient.Resource(gvr).Namespace(manifestObj.GetNamespace()).Create( ctx, manifestObj, metav1.CreateOptions{FieldManager: workFieldManagerName}) if err == nil { klog.V(2).InfoS("successfully created the manifest", "gvr", gvr, "manifest", manifestRef) + if _, err := r.createConfigMap(ctx, gvr, actual, owner, configMapName, "", ""); err != nil { + return nil, ManifestNoChangeAction, err + } return actual, ManifestCreatedAction, nil } return nil, ManifestNoChangeAction, err @@ -382,9 +384,34 @@ func (r *ApplyWorkReconciler) applyUnstructured(ctx context.Context, gvr schema. return nil, ManifestNoChangeAction, err } - // We only try to update the object if its spec hash value has changed. - if manifestObj.GetAnnotations()[manifestHashAnnotation] != curObj.GetAnnotations()[manifestHashAnnotation] { - return r.patchCurrentResource(ctx, gvr, manifestObj, curObj) + // get config map or create it if it doesn't exist + curObjConfigMap, err := r.getConfigMap(ctx, gvr, manifestObj, curObj, owner) + if err != nil { + return nil, ManifestNoChangeAction, err + } + + // set config map name annotation on manifest object to prevent deletion during patch. + setConfigMapNameAnnotation(curObjConfigMap.GetName(), manifestObj) + + if curObj.GetResourceVersion() != curObjConfigMap.Data[versionKey] { + // Need to update the config map, with current Object because the controller failed to update config map after patch + manifestHash, err := computeManifestHash(curObj) + if err != nil { + return nil, ManifestNoChangeAction, err + } + if err := r.updateConfigMap(ctx, curObjConfigMap, curObj, manifestHash); err != nil { + return nil, ManifestNoChangeAction, err + } + } + + manifestHash, err := computeManifestHash(manifestObj) + if err != nil { + return nil, ManifestNoChangeAction, err + } + + // We update the object if its spec hash value has changed or if config map got deleted or never created + if manifestHash != curObjConfigMap.Data[manifestHashKey] { + return r.patchCurrentResource(ctx, gvr, manifestObj, curObj, manifestHash, curObjConfigMap) } return curObj, ManifestNoChangeAction, nil @@ -392,24 +419,20 @@ func (r *ApplyWorkReconciler) applyUnstructured(ctx context.Context, gvr schema. // patchCurrentResource uses three way merge to patch the current resource with the new manifest we get from the work. func (r *ApplyWorkReconciler) patchCurrentResource(ctx context.Context, gvr schema.GroupVersionResource, - manifestObj, curObj *unstructured.Unstructured) (*unstructured.Unstructured, applyAction, error) { + manifestObj, curObj *unstructured.Unstructured, manifestHash string, curObjConfigMap *v1.ConfigMap) (*unstructured.Unstructured, applyAction, error) { manifestRef := klog.ObjectRef{ Name: manifestObj.GetName(), Namespace: manifestObj.GetNamespace(), } klog.V(2).InfoS("manifest is modified", "gvr", gvr, "manifest", manifestRef, - "new hash", manifestObj.GetAnnotations()[manifestHashAnnotation], - "existing hash", curObj.GetAnnotations()[manifestHashAnnotation]) + "new hash", manifestHash, + "existing hash", curObjConfigMap.Data[manifestHashKey]) // we need to merge the owner reference between the current and the manifest since we support one manifest // belong to multiple work so it contains the union of all the appliedWork manifestObj.SetOwnerReferences(mergeOwnerReference(curObj.GetOwnerReferences(), manifestObj.GetOwnerReferences())) - // record the raw manifest with the hash annotation in the manifest - if err := setModifiedConfigurationAnnotation(manifestObj); err != nil { - return nil, ManifestNoChangeAction, err - } // create the three-way merge patch between the current, original and manifest similar to how kubectl apply does - patch, err := threeWayMergePatch(curObj, manifestObj) + patch, err := threeWayMergePatch(curObj, manifestObj, curObjConfigMap) if err != nil { klog.ErrorS(err, "failed to generate the three way patch", "gvr", gvr, "manifest", manifestRef) return nil, ManifestNoChangeAction, err @@ -420,14 +443,22 @@ func (r *ApplyWorkReconciler) patchCurrentResource(ctx context.Context, gvr sche return nil, ManifestNoChangeAction, err } // Use client side apply the patch to the member cluster - manifestObj, patchErr := r.spokeDynamicClient.Resource(gvr).Namespace(manifestObj.GetNamespace()). + updatedManifestObj, patchErr := r.spokeDynamicClient.Resource(gvr).Namespace(manifestObj.GetNamespace()). Patch(ctx, manifestObj.GetName(), patch.Type(), data, metav1.PatchOptions{FieldManager: workFieldManagerName}) if patchErr != nil { klog.ErrorS(patchErr, "failed to patch the manifest", "gvr", gvr, "manifest", manifestRef) return nil, ManifestNoChangeAction, patchErr } + + //TODO: Need to handle case where configmap doesn't get updated + + // calculate the values based on obj from hub cluster, only track revisions from hub cluster changes + // passing in manifest hash calculated earlier as ownerRef field might have changed with mergeOwnerReference + if err := r.updateConfigMap(ctx, curObjConfigMap, manifestObj, manifestHash); err != nil { + return nil, ManifestNoChangeAction, err + } klog.V(2).InfoS("manifest patch succeeded", "gvr", gvr, "manifest", manifestRef) - return manifestObj, ManifestUpdatedAction, nil + return updatedManifestObj, ManifestUpdatedAction, nil } // generateWorkCondition constructs the work condition based on the apply result @@ -513,11 +544,10 @@ func (r *ApplyWorkReconciler) SetupWithManager(mgr ctrl.Manager) error { // we have modified. func computeManifestHash(obj *unstructured.Unstructured) (string, error) { manifest := obj.DeepCopy() - // remove the last applied Annotation to avoid unlimited recursion + // remove the config map annotation to avoid unlimited recursion, need to think ? annotation := manifest.GetAnnotations() if annotation != nil { - delete(annotation, manifestHashAnnotation) - delete(annotation, lastAppliedConfigAnnotation) + delete(annotation, configMapNameAnnotation) if len(annotation) == 0 { manifest.SetAnnotations(nil) } else { @@ -629,6 +659,209 @@ func buildManifestAppliedCondition(err error, action applyAction, observedGenera } } +func (r *ApplyWorkReconciler) createConfigMap(ctx context.Context, gvr schema.GroupVersionResource, manifestObj *unstructured.Unstructured, owner metav1.OwnerReference, configMapName, manifestHash, lastModifiedConfig string) (*v1.ConfigMap, error) { + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: fleetSystemNamespace, + }, + Data: map[string]string{}, + } + var err error + if manifestHash == "" { + manifestHash, err = computeManifestHash(manifestObj) + if err != nil { + return nil, err + } + } + if lastModifiedConfig == "" { + lastModifiedConfig, err = computeModifiedConfiguration(manifestObj) + if err != nil { + return nil, err + } + } + configMap.Data[manifestHashKey] = manifestHash + configMap.Data[lastAppliedConfigKey] = lastModifiedConfig + configMap.Data[versionKey] = manifestObj.GetResourceVersion() + // add applied work owner reference + addOwnerRef(owner, configMap) + // Annotating config map with manifest identifier to create two-way link. + setManifestIdentifierAnnotation(gvr, manifestObj, configMap) + if err := r.spokeClient.Create(ctx, configMap); err != nil { + if apierrors.IsAlreadyExists(err) { + // delete config map annotation from manifest + annotations := manifestObj.GetAnnotations() + delete(annotations, configMapName) + if _, err := r.spokeDynamicClient.Resource(gvr).Namespace(manifestObj.GetNamespace()).Update(ctx, manifestObj, metav1.UpdateOptions{FieldManager: workFieldManagerName}); err != nil { + klog.ErrorS(err, "failed to delete config map name annotation on manifest", "manifest", manifestObj.GetName(), "namespace", manifestObj.GetNamespace()) + return nil, err + } + klog.InfoS("config map annotation was removed from manifest since config map with same name already exists", "manifest", manifestObj.GetName(), "namespace", manifestObj.GetNamespace(), "configMap", configMapName) + } + klog.ErrorS(err, "config map create failed", "configMap", configMap.GetName()) + return nil, err + } + klog.V(2).InfoS("config map was created", "configMap", configMap.GetName()) + return configMap, nil +} + +// Need to handle case where manifest obj and configmap annotations +func (r *ApplyWorkReconciler) getConfigMap(ctx context.Context, gvr schema.GroupVersionResource, manifestObj, currentObj *unstructured.Unstructured, owner metav1.OwnerReference) (*v1.ConfigMap, error) { + var configMap v1.ConfigMap + var manifestHash, lastModifiedConfig string + + objAnnotations := currentObj.GetAnnotations() + if objAnnotations == nil { + klog.InfoS("manifest's annotation is empty") + objAnnotations = map[string]string{} + } else { + // Need to check and migrate annotations present on manifests to configmaps + existingManifestHash, existingLastModifiedConfig, err := getAndRemoveExistingAnnotation(currentObj) + if err != nil { + return nil, err + } + if existingManifestHash != "" && existingLastModifiedConfig != "" { + manifestHash = existingManifestHash + lastModifiedConfig = existingLastModifiedConfig + } + } + + configMapName, ok := objAnnotations[configMapNameAnnotation] + if !ok { + // Need to handle case where manifest has config map annotation removed due to conflict, it needs a new configmap name or old manifest objects won't have a config map associated with them + setConfigMapNameAnnotation(getRandomConfigMapName(), currentObj) + newConfigMap, err := r.updateCurrentObjectAndCreateConfigMap(ctx, gvr, currentObj, owner, manifestHash, lastModifiedConfig) + if err != nil { + return nil, err + } + return newConfigMap, nil + } + + // if configmap name annotation exists + if err := r.spokeClient.Get(ctx, types.NamespacedName{Name: configMapName, Namespace: fleetSystemNamespace}, &configMap); err != nil { + // config map doesn't exist + if apierrors.IsNotFound(err) { + // use existing config map name annotation, since the config map doesn't exist + newConfigMap, err := r.createConfigMap(ctx, gvr, currentObj, owner, configMapName, manifestHash, lastModifiedConfig) + if err != nil { + klog.ErrorS(err, "cannot create config map for manifest", "manifest", currentObj.GetName(), "namespace", currentObj.GetNamespace()) + return nil, err + } + return newConfigMap, err + } + klog.ErrorS(err, "failed to retrieve config map for manifest", "manifest", currentObj.GetName(), "config map", configMapName) + return nil, err + } + + // need to handle special case where manifest was created with config map name but controller failed without checking if config map already exists, turns out config map belonged to another manifest. + configMapAnnotations := configMap.GetAnnotations() + annotatedManifestIdentifier, ok := configMapAnnotations[manifestIdentifierAnnotation] + manifestIdentifier := getManifestIdentifier(gvr, manifestObj) + if annotatedManifestIdentifier != manifestIdentifier { + setConfigMapNameAnnotation(getRandomConfigMapName(), currentObj) + newConfigMap, err := r.updateCurrentObjectAndCreateConfigMap(ctx, gvr, currentObj, owner, manifestHash, lastModifiedConfig) + if err != nil { + return nil, err + } + return newConfigMap, nil + } + return &configMap, nil +} + +func (r *ApplyWorkReconciler) updateCurrentObjectAndCreateConfigMap(ctx context.Context, gvr schema.GroupVersionResource, currentObj *unstructured.Unstructured, + owner metav1.OwnerReference, manifestHash, lastModifiedConfig string) (*v1.ConfigMap, error) { + if err := r.spokeClient.Update(ctx, currentObj); err != nil { + return nil, err + } + curObjAnnotations := currentObj.GetAnnotations() + // we need to use current Object here since manifest Object is from hub cluster + newConfigMap, err := r.createConfigMap(ctx, gvr, currentObj, owner, curObjAnnotations[configMapNameAnnotation], manifestHash, lastModifiedConfig) + if err != nil { + klog.ErrorS(err, "cannot create config map for object", "manifest", currentObj.GetName(), "namespace", currentObj.GetNamespace()) + return nil, err + } + return newConfigMap, nil +} + +func (r *ApplyWorkReconciler) updateConfigMap(ctx context.Context, configMap *v1.ConfigMap, obj *unstructured.Unstructured, manifestHash string) error { + updatedLastModifiedConfig, err := computeModifiedConfiguration(obj) + if err != nil { + // namespace for manifest can be empty + klog.ErrorS(err, "failed to compute new last modified configuration", "manifest", obj.GetName(), "namespace", obj.GetNamespace()) + return err + } + configMap.Data[manifestHashKey] = manifestHash + configMap.Data[lastAppliedConfigKey] = updatedLastModifiedConfig + if err := r.spokeClient.Update(ctx, configMap); err != nil { + klog.ErrorS(err, "failed to update config map", "configMap", configMap.Name) + return err + } + return nil +} + +// We can calculate the manifestHash & lastAppliedConfig again wiping out previous state or make sure the annotations in the last applied config doesn't cause issue in patch. +func getAndRemoveExistingAnnotation(obj *unstructured.Unstructured) (string, string, error) { + objAnnotations := obj.GetAnnotations() + if objAnnotations == nil { + klog.InfoS("annotations is nil, no existing annotations to migrate", "manifest", obj.GetName(), "namespace", obj.GetNamespace()) + return "", "", nil + } + // Assuming that manifest has both annotations on it + manifestHash, ok1 := objAnnotations[manifestHashAnnotation] + lastModifiedConfig, ok2 := objAnnotations[lastAppliedConfigAnnotation] + if ok1 && ok2 { + klog.InfoS("annotation on manifest needs to be migrated", "manifest", obj.GetName(), "namespace", obj.GetNamespace()) + delete(objAnnotations, manifestHashAnnotation) + delete(objAnnotations, lastAppliedConfigAnnotation) + obj.SetAnnotations(objAnnotations) + return manifestHash, lastModifiedConfig, nil + } else if !ok1 && ok2 || ok1 && !ok2 { + return "", "", errors.New(fmt.Sprintf("manifest (%s/%s)cannot be migrated because it doesn't have manifestHash/lastModified Annotation", obj.GetName(), obj.GetNamespace())) + } + klog.InfoS("manifest doesn't contain annotations that need to be migrated", "manifest", obj.GetName(), "namespace", obj.GetNamespace()) + return "", "", nil +} + +func getRandomConfigMapName() string { + generator := names.SimpleNameGenerator + return generator.GenerateName(configMapNamePrefix) +} + +func setConfigMapNameAnnotation(configMapName string, obj *unstructured.Unstructured) { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[configMapNameAnnotation] = configMapName + obj.SetAnnotations(annotations) +} + +func setManifestIdentifierAnnotation(gvr schema.GroupVersionResource, obj *unstructured.Unstructured, configMap *v1.ConfigMap) { + manifestId := getManifestIdentifier(gvr, obj) + annotations := configMap.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[manifestIdentifierAnnotation] = manifestId + configMap.SetAnnotations(annotations) +} + +func getManifestIdentifier(gvr schema.GroupVersionResource, obj *unstructured.Unstructured) string { + var manifestId, groupVersion string + isNamespaced := len(obj.GetNamespace()) > 0 + if gvr.Group != "" { + groupVersion = gvr.Group + "-" + gvr.Version + } else { + groupVersion = gvr.Version + } + if isNamespaced { + manifestId = groupVersion + "-" + gvr.Resource + "-" + obj.GetName() + "-" + obj.GetNamespace() + } else { + manifestId = groupVersion + "-" + gvr.Resource + "-" + obj.GetName() + } + return manifestId +} + // generateWorkAppliedCondition generate applied status condition for work. // If one of the manifests is applied failed on the spoke, the applied status condition of the work is false. func generateWorkAppliedCondition(manifestConditions []workv1alpha1.ManifestCondition, observedGeneration int64) metav1.Condition { diff --git a/pkg/controllers/work/manager.go b/pkg/controllers/work/manager.go index 067471bc7..061a5bda4 100644 --- a/pkg/controllers/work/manager.go +++ b/pkg/controllers/work/manager.go @@ -32,9 +32,14 @@ import ( const ( workFinalizer = "fleet.azure.com/work-cleanup" - manifestHashAnnotation = "fleet.azure.com/spec-hash" - - lastAppliedConfigAnnotation = "fleet.azure.com/last-applied-configuration" + configMapNameAnnotation = "fleet.azure.com/config-map-name" + manifestIdentifierAnnotation = "fleet.azure.com/manifest-identifier" + manifestHashAnnotation = "fleet.azure.com/spec-hash" + lastAppliedConfigAnnotation = "fleet.azure.com/last-applied-configuration" + + manifestHashKey = "fleet-spec-hash" + lastAppliedConfigKey = "fleet-last-applied-configuration" + versionKey = "fleet-version-key" ConditionTypeApplied = "Applied" ConditionTypeAvailable = "Available" diff --git a/pkg/controllers/work/patch_util.go b/pkg/controllers/work/patch_util.go index 10d433b1d..0bc7fc119 100644 --- a/pkg/controllers/work/patch_util.go +++ b/pkg/controllers/work/patch_util.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -25,17 +26,14 @@ func init() { // threeWayMergePatch creates a patch by computing a three-way diff based on // an object's current state, modified state, and last-applied-state recorded in its annotation. -func threeWayMergePatch(currentObj, manifestObj client.Object) (client.Patch, error) { +func threeWayMergePatch(currentObj, manifestObj client.Object, currentObjConfigMap *v1.ConfigMap) (client.Patch, error) { //TODO: see if we should use something like json.ConfigCompatibleWithStandardLibrary.Marshal to make sure that // the json we created is compatible with the format that json merge patch requires. current, err := json.Marshal(currentObj) if err != nil { return nil, err } - original, err := getOriginalConfiguration(currentObj) - if err != nil { - return nil, err - } + original := []byte(currentObjConfigMap.Data[lastAppliedConfigKey]) manifest, err := json.Marshal(manifestObj) if err != nil { return nil, err @@ -109,6 +107,15 @@ func setModifiedConfigurationAnnotation(obj runtime.Object) error { return metadataAccessor.SetAnnotations(obj, annotations) } +// computeModifiedConfiguration computes the json marshal and returns it. +func computeModifiedConfiguration(obj runtime.Object) (string, error) { + modified, err := json.Marshal(obj) + if err != nil { + return "", err + } + return string(modified), nil +} + // getOriginalConfiguration gets original configuration of the object // form the annotation, or return an error if no annotation found. func getOriginalConfiguration(obj runtime.Object) ([]byte, error) {