From db51e51fa0616e545b465e57c9883212155105c7 Mon Sep 17 00:00:00 2001 From: MichaluxPL <68371308+MichaluxPL@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:05:39 +0200 Subject: [PATCH 01/12] feat: add sidecar resources definition to ObjectStore CRD Modify ObjectStore CRD to enable resources specification for the plugin's sidecar container. Signed-off-by: MichaluxPL <68371308+MichaluxPL@users.noreply.github.com> --- .../barmancloud.cnpg.io_objectstores.yaml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml b/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml index f55fda46..2dce3871 100644 --- a/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml +++ b/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml @@ -517,6 +517,37 @@ spec: The retentionCheckInterval defines the frequency at which the system checks and enforces retention policies. type: integer + resources: + description: |- + Plugin's sidecar resources requirements. Please refer to + https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + for more information. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object type: object retentionPolicy: description: |- From 71a188e97eca317609d953d04cb229fe068431fd Mon Sep 17 00:00:00 2001 From: MichaluxPL <68371308+MichaluxPL@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:12:37 +0200 Subject: [PATCH 02/12] feat: add sidecar resources definition in InstanceSidecarConfiguration stanza Add sidecar resources definition in InstanceSidecarConfiguration stanza Signed-off-by: MichaluxPL <68371308+MichaluxPL@users.noreply.github.com> --- api/v1/objectstore_types.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/v1/objectstore_types.go b/api/v1/objectstore_types.go index 0b15d71d..1789e4f6 100644 --- a/api/v1/objectstore_types.go +++ b/api/v1/objectstore_types.go @@ -33,6 +33,9 @@ type InstanceSidecarConfiguration struct { // +kubebuilder:default:=1800 // +optional RetentionPolicyIntervalSeconds int `json:"retentionPolicyIntervalSeconds,omitempty"` + // Resources define cpu/memory requests and limits for the sidecar that runs in the instance pods. + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` } // ObjectStoreSpec defines the desired state of ObjectStore. From 0cf55b8398224f17321b11370cf9932fa1e8d8c1 Mon Sep 17 00:00:00 2001 From: MichaluxPL <68371308+MichaluxPL@users.noreply.github.com> Date: Tue, 22 Apr 2025 14:25:31 +0200 Subject: [PATCH 03/12] feat: update reconcile functions in lifecycle.go to add resources definition for the sidecar Update reconcile functions in lifecycle.go to add resources definition for the sidecar container. Signed-off-by: MichaluxPL <68371308+MichaluxPL@users.noreply.github.com> --- internal/cnpgi/operator/lifecycle.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/cnpgi/operator/lifecycle.go b/internal/cnpgi/operator/lifecycle.go index 2862da74..bccebef0 100644 --- a/internal/cnpgi/operator/lifecycle.go +++ b/internal/cnpgi/operator/lifecycle.go @@ -139,6 +139,7 @@ func reconcileJob( return nil, nil } + var barmanObjectStore barmancloudv1.ObjectStore var job batchv1.Job if err := decoder.DecodeObjectStrict( request.GetObjectDefinition(), @@ -171,6 +172,7 @@ func reconcileJob( }, env, certificates, + barmanObjectStore, ); err != nil { return nil, fmt.Errorf("while reconciling pod spec for job: %w", err) } @@ -202,7 +204,13 @@ func (impl LifecycleImplementation) reconcilePod( return nil, err } - return reconcilePod(ctx, cluster, request, pluginConfiguration, env, certificates) + var barmanObjectStore barmancloudv1.ObjectStore + configuration := config.NewFromCluster(cluster) + if err := impl.Client.Get(ctx, configuration.GetBarmanObjectKey(), &barmanObjectStore); err != nil { + return nil, err + } + + return reconcilePod(ctx, cluster, request, pluginConfiguration, env, certificates, barmanObjectStore) } func reconcilePod( @@ -212,6 +220,7 @@ func reconcilePod( pluginConfiguration *config.PluginConfiguration, env []corev1.EnvVar, certificates []corev1.VolumeProjection, + barmanObjectStore barmancloudv1.ObjectStore, ) (*lifecycle.OperatorLifecycleResponse, error) { pod, err := decoder.DecodePodJSON(request.GetObjectDefinition()) if err != nil { @@ -234,6 +243,7 @@ func reconcilePod( }, env, certificates, + barmanObjectStore, ); err != nil { return nil, fmt.Errorf("while reconciling pod spec for pod: %w", err) } @@ -259,6 +269,7 @@ func reconcilePodSpec( sidecarConfig corev1.Container, additionalEnvs []corev1.EnvVar, certificates []corev1.VolumeProjection, + barmanObjectStore barmancloudv1.ObjectStore, ) error { envs := []corev1.EnvVar{ { @@ -314,6 +325,7 @@ func reconcilePodSpec( Drop: []corev1.Capability{"ALL"}, }, } + sidecarConfig.Resources = barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources // merge the main container envs if they aren't already set for _, container := range spec.Containers { From 91e98994d20548f2017efbabbb690359de63228f Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Mon, 5 May 2025 14:19:11 +0200 Subject: [PATCH 04/12] chore: review Signed-off-by: Leonardo Cecchi --- .wordlist.txt | 2 + api/v1/objectstore_types.go | 5 +- api/v1/zz_generated.deepcopy.go | 1 + .../barmancloud.cnpg.io_objectstores.yaml | 49 ++++++++-- hack/examples/minio-store.yaml | 7 ++ internal/cnpgi/operator/lifecycle.go | 91 ++++++++++++------- internal/cnpgi/operator/lifecycle_test.go | 12 +-- manifest.yaml | 60 ++++++++++++ web/docs/plugin-barman-cloud.v1.md | 1 + 9 files changed, 175 insertions(+), 53 deletions(-) diff --git a/.wordlist.txt b/.wordlist.txt index a634c071..0f035f0d 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -33,6 +33,7 @@ README RPO RTO RecoveryWindow +ResourceRequirements RetentionPolicy SAS SFO @@ -64,6 +65,7 @@ cmctl cnpg codebase containerPort +cpu creds csi customresourcedefinition diff --git a/api/v1/objectstore_types.go b/api/v1/objectstore_types.go index 1789e4f6..80c4742a 100644 --- a/api/v1/objectstore_types.go +++ b/api/v1/objectstore_types.go @@ -33,9 +33,10 @@ type InstanceSidecarConfiguration struct { // +kubebuilder:default:=1800 // +optional RetentionPolicyIntervalSeconds int `json:"retentionPolicyIntervalSeconds,omitempty"` + // Resources define cpu/memory requests and limits for the sidecar that runs in the instance pods. - // +optional - Resources corev1.ResourceRequirements `json:"resources,omitempty"` + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` } // ObjectStoreSpec defines the desired state of ObjectStore. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index c70da27a..11fb2aea 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -35,6 +35,7 @@ func (in *InstanceSidecarConfiguration) DeepCopyInto(out *InstanceSidecarConfigu (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.Resources.DeepCopyInto(&out.Resources) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstanceSidecarConfiguration. diff --git a/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml b/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml index 2dce3871..6fb87b51 100644 --- a/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml +++ b/config/crd/bases/barmancloud.cnpg.io_objectstores.yaml @@ -511,18 +511,41 @@ spec: - name type: object type: array - retentionPolicyIntervalSeconds: - default: 1800 - description: |- - The retentionCheckInterval defines the frequency at which the - system checks and enforces retention policies. - type: integer resources: - description: |- - Plugin's sidecar resources requirements. Please refer to - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ - for more information. + description: Resources define cpu/memory requests and limits for + the sidecar that runs in the instance pods. properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -548,6 +571,12 @@ spec: More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ type: object type: object + retentionPolicyIntervalSeconds: + default: 1800 + description: |- + The retentionCheckInterval defines the frequency at which the + system checks and enforces retention policies. + type: integer type: object retentionPolicy: description: |- diff --git a/hack/examples/minio-store.yaml b/hack/examples/minio-store.yaml index 38b116f6..6c420d93 100644 --- a/hack/examples/minio-store.yaml +++ b/hack/examples/minio-store.yaml @@ -6,6 +6,13 @@ spec: retentionPolicy: "1m" instanceSidecarConfiguration: retentionPolicyIntervalSeconds: 30 + resources: + requests: + memory: "64Mi" + cpu: "250m" + limits: + memory: "128Mi" + cpu: "500m" configuration: endpointCA: name: minio-server-tls diff --git a/internal/cnpgi/operator/lifecycle.go b/internal/cnpgi/operator/lifecycle.go index bccebef0..dcfefa24 100644 --- a/internal/cnpgi/operator/lifecycle.go +++ b/internal/cnpgi/operator/lifecycle.go @@ -17,6 +17,7 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/config" ) @@ -123,15 +124,29 @@ func (impl LifecycleImplementation) reconcileJob( return nil, err } - return reconcileJob(ctx, cluster, request, env, certificates) + resources, err := impl.collectSidecarResources(ctx, pluginConfiguration) + if err != nil { + return nil, err + } + + return reconcileJob(ctx, cluster, request, sidecarConfiguration{ + env: env, + certificates: certificates, + resources: resources, + }) +} + +type sidecarConfiguration struct { + env []corev1.EnvVar + certificates []corev1.VolumeProjection + resources corev1.ResourceRequirements } func reconcileJob( ctx context.Context, cluster *cnpgv1.Cluster, request *lifecycle.OperatorLifecycleRequest, - env []corev1.EnvVar, - certificates []corev1.VolumeProjection, + config sidecarConfiguration, ) (*lifecycle.OperatorLifecycleResponse, error) { contextLogger := log.FromContext(ctx).WithName("lifecycle") if pluginConfig := cluster.GetRecoverySourcePlugin(); pluginConfig == nil || pluginConfig.Name != metadata.PluginName { @@ -139,7 +154,6 @@ func reconcileJob( return nil, nil } - var barmanObjectStore barmancloudv1.ObjectStore var job batchv1.Job if err := decoder.DecodeObjectStrict( request.GetObjectDefinition(), @@ -170,9 +184,7 @@ func reconcileJob( corev1.Container{ Args: []string{"restore"}, }, - env, - certificates, - barmanObjectStore, + config, ); err != nil { return nil, fmt.Errorf("while reconciling pod spec for job: %w", err) } @@ -204,13 +216,28 @@ func (impl LifecycleImplementation) reconcilePod( return nil, err } + resources, err := impl.collectSidecarResources(ctx, pluginConfiguration) + if err != nil { + return nil, err + } + + return reconcilePod(ctx, cluster, request, pluginConfiguration, sidecarConfiguration{ + env: env, + certificates: certificates, + resources: resources, + }) +} + +func (impl LifecycleImplementation) collectSidecarResources( + ctx context.Context, + configuration *config.PluginConfiguration, +) (corev1.ResourceRequirements, error) { var barmanObjectStore barmancloudv1.ObjectStore - configuration := config.NewFromCluster(cluster) - if err := impl.Client.Get(ctx, configuration.GetBarmanObjectKey(), &barmanObjectStore); err != nil { - return nil, err - } + if err := impl.Client.Get(ctx, configuration.GetBarmanObjectKey(), &barmanObjectStore); err != nil { + return corev1.ResourceRequirements{}, err + } - return reconcilePod(ctx, cluster, request, pluginConfiguration, env, certificates, barmanObjectStore) + return barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources, nil } func reconcilePod( @@ -218,9 +245,7 @@ func reconcilePod( cluster *cnpgv1.Cluster, request *lifecycle.OperatorLifecycleRequest, pluginConfiguration *config.PluginConfiguration, - env []corev1.EnvVar, - certificates []corev1.VolumeProjection, - barmanObjectStore barmancloudv1.ObjectStore, + config sidecarConfiguration, ) (*lifecycle.OperatorLifecycleResponse, error) { pod, err := decoder.DecodePodJSON(request.GetObjectDefinition()) if err != nil { @@ -241,9 +266,7 @@ func reconcilePod( corev1.Container{ Args: []string{"instance"}, }, - env, - certificates, - barmanObjectStore, + config, ); err != nil { return nil, fmt.Errorf("while reconciling pod spec for pod: %w", err) } @@ -266,10 +289,8 @@ func reconcilePodSpec( cluster *cnpgv1.Cluster, spec *corev1.PodSpec, mainContainerName string, - sidecarConfig corev1.Container, - additionalEnvs []corev1.EnvVar, - certificates []corev1.VolumeProjection, - barmanObjectStore barmancloudv1.ObjectStore, + sidecarTemplate corev1.Container, + config sidecarConfiguration, ) error { envs := []corev1.EnvVar{ { @@ -296,7 +317,7 @@ func reconcilePodSpec( }, } - envs = append(envs, additionalEnvs...) + envs = append(envs, config.env...) baseProbe := &corev1.Probe{ FailureThreshold: 10, @@ -309,11 +330,11 @@ func reconcilePodSpec( } // fixed values - sidecarConfig.Name = "plugin-barman-cloud" - sidecarConfig.Image = viper.GetString("sidecar-image") - sidecarConfig.ImagePullPolicy = cluster.Spec.ImagePullPolicy - sidecarConfig.StartupProbe = baseProbe.DeepCopy() - sidecarConfig.SecurityContext = &corev1.SecurityContext{ + sidecarTemplate.Name = "plugin-barman-cloud" + sidecarTemplate.Image = viper.GetString("sidecar-image") + sidecarTemplate.ImagePullPolicy = cluster.Spec.ImagePullPolicy + sidecarTemplate.StartupProbe = baseProbe.DeepCopy() + sidecarTemplate.SecurityContext = &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(false), RunAsNonRoot: ptr.To(true), Privileged: ptr.To(false), @@ -325,21 +346,21 @@ func reconcilePodSpec( Drop: []corev1.Capability{"ALL"}, }, } - sidecarConfig.Resources = barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources + sidecarTemplate.Resources = config.resources // merge the main container envs if they aren't already set for _, container := range spec.Containers { if container.Name == mainContainerName { for _, env := range container.Env { found := false - for _, existingEnv := range sidecarConfig.Env { + for _, existingEnv := range sidecarTemplate.Env { if existingEnv.Name == env.Name { found = true break } } if !found { - sidecarConfig.Env = append(sidecarConfig.Env, env) + sidecarTemplate.Env = append(sidecarTemplate.Env, env) } } break @@ -349,18 +370,18 @@ func reconcilePodSpec( // merge the default envs if they aren't already set for _, env := range envs { found := false - for _, existingEnv := range sidecarConfig.Env { + for _, existingEnv := range sidecarTemplate.Env { if existingEnv.Name == env.Name { found = true break } } if !found { - sidecarConfig.Env = append(sidecarConfig.Env, env) + sidecarTemplate.Env = append(sidecarTemplate.Env, env) } } - if err := injectPluginSidecarPodSpec(spec, &sidecarConfig, mainContainerName); err != nil { + if err := injectPluginSidecarPodSpec(spec, &sidecarTemplate, mainContainerName); err != nil { return err } @@ -370,7 +391,7 @@ func reconcilePodSpec( Name: barmanCertificatesVolumeName, VolumeSource: corev1.VolumeSource{ Projected: &corev1.ProjectedVolumeSource{ - Sources: certificates, + Sources: config.certificates, }, }, }) diff --git a/internal/cnpgi/operator/lifecycle_test.go b/internal/cnpgi/operator/lifecycle_test.go index 1e63b6f2..3d94a3b6 100644 --- a/internal/cnpgi/operator/lifecycle_test.go +++ b/internal/cnpgi/operator/lifecycle_test.go @@ -107,7 +107,7 @@ var _ = Describe("LifecycleImplementation", func() { ObjectDefinition: jobJSON, } - response, err := reconcileJob(ctx, cluster, request, nil, nil) + response, err := reconcileJob(ctx, cluster, request, sidecarConfiguration{}) Expect(err).NotTo(HaveOccurred()) Expect(response).NotTo(BeNil()) Expect(response.JsonPatch).NotTo(BeEmpty()) @@ -128,7 +128,7 @@ var _ = Describe("LifecycleImplementation", func() { ObjectDefinition: jobJSON, } - response, err := reconcileJob(ctx, cluster, request, nil, nil) + response, err := reconcileJob(ctx, cluster, request, sidecarConfiguration{}) Expect(err).NotTo(HaveOccurred()) Expect(response).To(BeNil()) }) @@ -138,7 +138,7 @@ var _ = Describe("LifecycleImplementation", func() { ObjectDefinition: []byte("invalid-json"), } - response, err := reconcileJob(ctx, cluster, request, nil, nil) + response, err := reconcileJob(ctx, cluster, request, sidecarConfiguration{}) Expect(err).To(HaveOccurred()) Expect(response).To(BeNil()) }) @@ -165,7 +165,7 @@ var _ = Describe("LifecycleImplementation", func() { ObjectDefinition: jobJSON, } - response, err := reconcileJob(ctx, cluster, request, nil, nil) + response, err := reconcileJob(ctx, cluster, request, sidecarConfiguration{}) Expect(err).NotTo(HaveOccurred()) Expect(response).To(BeNil()) }) @@ -185,7 +185,7 @@ var _ = Describe("LifecycleImplementation", func() { ObjectDefinition: podJSON, } - response, err := reconcilePod(ctx, cluster, request, pluginConfiguration, nil, nil) + response, err := reconcilePod(ctx, cluster, request, pluginConfiguration, sidecarConfiguration{}) Expect(err).NotTo(HaveOccurred()) Expect(response).NotTo(BeNil()) Expect(response.JsonPatch).NotTo(BeEmpty()) @@ -203,7 +203,7 @@ var _ = Describe("LifecycleImplementation", func() { ObjectDefinition: []byte("invalid-json"), } - response, err := reconcilePod(ctx, cluster, request, pluginConfiguration, nil, nil) + response, err := reconcilePod(ctx, cluster, request, pluginConfiguration, sidecarConfiguration{}) Expect(err).To(HaveOccurred()) Expect(response).To(BeNil()) }) diff --git a/manifest.yaml b/manifest.yaml index e19a59d3..c43ef4b5 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -510,6 +510,66 @@ spec: - name type: object type: array + resources: + description: Resources define cpu/memory requests and limits for + the sidecar that runs in the instance pods. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object retentionPolicyIntervalSeconds: default: 1800 description: |- diff --git a/web/docs/plugin-barman-cloud.v1.md b/web/docs/plugin-barman-cloud.v1.md index 197c1edf..552dc274 100644 --- a/web/docs/plugin-barman-cloud.v1.md +++ b/web/docs/plugin-barman-cloud.v1.md @@ -28,6 +28,7 @@ _Appears in:_ | --- | --- | --- | --- | --- | | `env` _[EnvVar](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#envvar-v1-core) array_ | The environment to be explicitly passed to the sidecar | | | | | `retentionPolicyIntervalSeconds` _integer_ | The retentionCheckInterval defines the frequency at which the
system checks and enforces retention policies. | | 1800 | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | Resources define cpu/memory requests and limits for the sidecar that runs in the instance pods. | | | | #### ObjectStore From 293c9e901f5171cd6612588c45fc8224421dc254 Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Tue, 6 May 2025 10:50:54 +0200 Subject: [PATCH 05/12] feat: get resources from recovery object store Signed-off-by: Leonardo Cecchi --- hack/examples/minio-store.yaml | 2 +- internal/cnpgi/operator/lifecycle.go | 17 +------ .../cnpgi/operator/lifecycle_certificates.go | 15 ++++++ internal/cnpgi/operator/lifecycle_envs.go | 17 +++++++ .../cnpgi/operator/lifecycle_resources.go | 49 +++++++++++++++++++ 5 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 internal/cnpgi/operator/lifecycle_resources.go diff --git a/hack/examples/minio-store.yaml b/hack/examples/minio-store.yaml index 6c420d93..6304a929 100644 --- a/hack/examples/minio-store.yaml +++ b/hack/examples/minio-store.yaml @@ -11,7 +11,7 @@ spec: memory: "64Mi" cpu: "250m" limits: - memory: "128Mi" + memory: "512Mi" cpu: "500m" configuration: endpointCA: diff --git a/internal/cnpgi/operator/lifecycle.go b/internal/cnpgi/operator/lifecycle.go index dcfefa24..308c9e7e 100644 --- a/internal/cnpgi/operator/lifecycle.go +++ b/internal/cnpgi/operator/lifecycle.go @@ -17,7 +17,6 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" - barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/metadata" "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/config" ) @@ -124,7 +123,7 @@ func (impl LifecycleImplementation) reconcileJob( return nil, err } - resources, err := impl.collectSidecarResources(ctx, pluginConfiguration) + resources, err := impl.collectSidecarResourcesForRecoveryJob(ctx, pluginConfiguration) if err != nil { return nil, err } @@ -216,7 +215,7 @@ func (impl LifecycleImplementation) reconcilePod( return nil, err } - resources, err := impl.collectSidecarResources(ctx, pluginConfiguration) + resources, err := impl.collectSidecarResourcesForPod(ctx, pluginConfiguration) if err != nil { return nil, err } @@ -228,18 +227,6 @@ func (impl LifecycleImplementation) reconcilePod( }) } -func (impl LifecycleImplementation) collectSidecarResources( - ctx context.Context, - configuration *config.PluginConfiguration, -) (corev1.ResourceRequirements, error) { - var barmanObjectStore barmancloudv1.ObjectStore - if err := impl.Client.Get(ctx, configuration.GetBarmanObjectKey(), &barmanObjectStore); err != nil { - return corev1.ResourceRequirements{}, err - } - - return barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources, nil -} - func reconcilePod( ctx context.Context, cluster *cnpgv1.Cluster, diff --git a/internal/cnpgi/operator/lifecycle_certificates.go b/internal/cnpgi/operator/lifecycle_certificates.go index 800612ec..5ff3e0b8 100644 --- a/internal/cnpgi/operator/lifecycle_certificates.go +++ b/internal/cnpgi/operator/lifecycle_certificates.go @@ -30,6 +30,21 @@ func (impl LifecycleImplementation) collectAdditionalCertificates( result = append(result, certs...) } + if len(pluginConfiguration.ReplicaSourceBarmanObjectName) > 0 && + pluginConfiguration.ReplicaSourceBarmanObjectName != pluginConfiguration.BarmanObjectName { + envs, err := impl.collectObjectStoreCertificates( + ctx, + types.NamespacedName{ + Name: pluginConfiguration.RecoveryBarmanObjectName, + Namespace: namespace, + }, + ) + if err != nil { + return nil, err + } + result = append(result, envs...) + } + return result, nil } diff --git a/internal/cnpgi/operator/lifecycle_envs.go b/internal/cnpgi/operator/lifecycle_envs.go index bfb5b226..e51b5c8b 100644 --- a/internal/cnpgi/operator/lifecycle_envs.go +++ b/internal/cnpgi/operator/lifecycle_envs.go @@ -17,6 +17,9 @@ func (impl LifecycleImplementation) collectAdditionalEnvs( ) ([]corev1.EnvVar, error) { var result []corev1.EnvVar + // TODO: check if the environment variables are clashing and in + // that case raise an error + if len(pluginConfiguration.BarmanObjectName) > 0 { envs, err := impl.collectObjectStoreEnvs( ctx, @@ -45,6 +48,20 @@ func (impl LifecycleImplementation) collectAdditionalEnvs( result = append(result, envs...) } + if len(pluginConfiguration.ReplicaSourceBarmanObjectName) > 0 { + envs, err := impl.collectObjectStoreEnvs( + ctx, + types.NamespacedName{ + Name: pluginConfiguration.ReplicaSourceBarmanObjectName, + Namespace: namespace, + }, + ) + if err != nil { + return nil, err + } + result = append(result, envs...) + } + return result, nil } diff --git a/internal/cnpgi/operator/lifecycle_resources.go b/internal/cnpgi/operator/lifecycle_resources.go new file mode 100644 index 00000000..52438f61 --- /dev/null +++ b/internal/cnpgi/operator/lifecycle_resources.go @@ -0,0 +1,49 @@ +package operator + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + + barmancloudv1 "github.com/cloudnative-pg/plugin-barman-cloud/api/v1" + "github.com/cloudnative-pg/plugin-barman-cloud/internal/cnpgi/operator/config" +) + +func (impl LifecycleImplementation) collectSidecarResourcesForRecoveryJob( + ctx context.Context, + configuration *config.PluginConfiguration, +) (corev1.ResourceRequirements, error) { + if len(configuration.RecoveryBarmanObjectName) > 0 { + var barmanObjectStore barmancloudv1.ObjectStore + if err := impl.Client.Get(ctx, configuration.GetRecoveryBarmanObjectKey(), &barmanObjectStore); err != nil { + return corev1.ResourceRequirements{}, err + } + + return barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources, nil + } + + return corev1.ResourceRequirements{}, nil +} + +func (impl LifecycleImplementation) collectSidecarResourcesForPod( + ctx context.Context, + configuration *config.PluginConfiguration, +) (corev1.ResourceRequirements, error) { + if len(configuration.BarmanObjectName) > 0 { + // On a replica cluster, the designated primary will use both the + // replica source object store and the object store of the cluster. + // The designed primary role can change without the Pod being + // recreated. + // In this case, we use the cluster object store for configuring + // the resources created by the sidecar. + + var barmanObjectStore barmancloudv1.ObjectStore + if err := impl.Client.Get(ctx, configuration.GetBarmanObjectKey(), &barmanObjectStore); err != nil { + return corev1.ResourceRequirements{}, err + } + + return barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources, nil + } + + return corev1.ResourceRequirements{}, nil +} From eae195b5d3b3a828b1110edf3ed84b6ec223a955 Mon Sep 17 00:00:00 2001 From: Marco Nenciarini Date: Thu, 8 May 2025 15:05:56 +0200 Subject: [PATCH 06/12] feat: fall back on recovery source object if cluster is not defined Signed-off-by: Marco Nenciarini --- .../cnpgi/operator/lifecycle_resources.go | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/internal/cnpgi/operator/lifecycle_resources.go b/internal/cnpgi/operator/lifecycle_resources.go index 52438f61..a68339b8 100644 --- a/internal/cnpgi/operator/lifecycle_resources.go +++ b/internal/cnpgi/operator/lifecycle_resources.go @@ -30,12 +30,11 @@ func (impl LifecycleImplementation) collectSidecarResourcesForPod( configuration *config.PluginConfiguration, ) (corev1.ResourceRequirements, error) { if len(configuration.BarmanObjectName) > 0 { - // On a replica cluster, the designated primary will use both the - // replica source object store and the object store of the cluster. - // The designed primary role can change without the Pod being - // recreated. + // On a replica cluster that also archives, the designated primary + // will use both the replica source object store and the object store + // of the cluster. // In this case, we use the cluster object store for configuring - // the resources created by the sidecar. + // the resources of the sidecar container. var barmanObjectStore barmancloudv1.ObjectStore if err := impl.Client.Get(ctx, configuration.GetBarmanObjectKey(), &barmanObjectStore); err != nil { @@ -45,5 +44,18 @@ func (impl LifecycleImplementation) collectSidecarResourcesForPod( return barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources, nil } + if len(configuration.RecoveryBarmanObjectName) > 0 { + // On a replica cluster that doesn't archive, the designated primary + // uses only the replica source object store. + // In this case, we use the replica source object store for configuring + // the resources of the sidecar container. + var barmanObjectStore barmancloudv1.ObjectStore + if err := impl.Client.Get(ctx, configuration.GetRecoveryBarmanObjectKey(), &barmanObjectStore); err != nil { + return corev1.ResourceRequirements{}, err + } + + return barmanObjectStore.Spec.InstanceSidecarConfiguration.Resources, nil + } + return corev1.ResourceRequirements{}, nil } From fd6f3e9b10122e4b40a458f164dcd1c25663ec4a Mon Sep 17 00:00:00 2001 From: Francesco Canovai Date: Fri, 9 May 2025 12:12:16 +0200 Subject: [PATCH 07/12] docs: describe instanceSidecarConfiguration Document how the instance sidecar is configured and what happens when resources definitions conflict between objectstores. Signed-off-by: Francesco Canovai --- web/docs/usage.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/web/docs/usage.md b/web/docs/usage.md index 1c464e86..9521c0d3 100644 --- a/web/docs/usage.md +++ b/web/docs/usage.md @@ -210,3 +210,37 @@ spec: parameters: barmanObjectName: minio-store-b ``` + +## Configuring the plugin instance sidecar + +The Barman Cloud Plugin uses a sidecar container that runs alongside each +PostgreSQL instance pod. This sidecar handles backup, WAL archiving, and restore +operations. You can control how the sidecar works by setting the +`.spec.instanceSidecarConfiguration` section in your `ObjectStore` resource. +These settings apply to all PostgreSQL instances that use this object store. + +```yaml +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: minio-store +spec: + configuration: + # [...] + instanceSidecarConfiguration: + retentionPolicyIntervalSeconds: 30 + resources: + requests: + memory: "64Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" +``` + +When the plugin is enabled, the sidecar is automatically injected into each +PostgreSQL instance pod. Even if you define multiple `ObjectStore` resources, +only one sidecar will run per instance. If a replica cluster also archives WALs +to a different `ObjectStore`, the sidecar will use the resource settings from the +`ObjectStore` referenced by the archiving plugin, not the one in the +`.spec.externalClusters` section. From 86e7377ef962764a8636399726a1900c1acc9d23 Mon Sep 17 00:00:00 2001 From: Francesco Canovai Date: Fri, 9 May 2025 12:12:16 +0200 Subject: [PATCH 08/12] docs: describe instanceSidecarConfiguration Document how the instance sidecar is configured and what happens when resources definitions conflict between objectstores. Signed-off-by: Francesco Canovai --- hack/examples/minio-store.yaml | 3 +-- web/docs/usage.md | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hack/examples/minio-store.yaml b/hack/examples/minio-store.yaml index 6304a929..de47ea7a 100644 --- a/hack/examples/minio-store.yaml +++ b/hack/examples/minio-store.yaml @@ -5,7 +5,7 @@ metadata: spec: retentionPolicy: "1m" instanceSidecarConfiguration: - retentionPolicyIntervalSeconds: 30 + retentionPolicyIntervalSeconds: 1800 resources: requests: memory: "64Mi" @@ -34,4 +34,3 @@ spec: - "--min-chunk-size=5MB" - "--read-timeout=60" - "-vv" - diff --git a/web/docs/usage.md b/web/docs/usage.md index 9521c0d3..a8bdd729 100644 --- a/web/docs/usage.md +++ b/web/docs/usage.md @@ -228,7 +228,7 @@ spec: configuration: # [...] instanceSidecarConfiguration: - retentionPolicyIntervalSeconds: 30 + retentionPolicyIntervalSeconds: 1800 resources: requests: memory: "64Mi" @@ -243,4 +243,5 @@ PostgreSQL instance pod. Even if you define multiple `ObjectStore` resources, only one sidecar will run per instance. If a replica cluster also archives WALs to a different `ObjectStore`, the sidecar will use the resource settings from the `ObjectStore` referenced by the archiving plugin, not the one in the -`.spec.externalClusters` section. +`.spec.externalClusters` section. Changes to the sidecar configuration will be +applied on the next reconciliation of the `Cluster` resources it. From d72adbe8101cdbe0ca77a19d65736249fbccce7c Mon Sep 17 00:00:00 2001 From: Francesco Canovai Date: Fri, 9 May 2025 16:04:08 +0200 Subject: [PATCH 09/12] fix: rebase Signed-off-by: Francesco Canovai --- internal/cnpgi/operator/lifecycle_certificates.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/internal/cnpgi/operator/lifecycle_certificates.go b/internal/cnpgi/operator/lifecycle_certificates.go index 5ff3e0b8..800612ec 100644 --- a/internal/cnpgi/operator/lifecycle_certificates.go +++ b/internal/cnpgi/operator/lifecycle_certificates.go @@ -30,21 +30,6 @@ func (impl LifecycleImplementation) collectAdditionalCertificates( result = append(result, certs...) } - if len(pluginConfiguration.ReplicaSourceBarmanObjectName) > 0 && - pluginConfiguration.ReplicaSourceBarmanObjectName != pluginConfiguration.BarmanObjectName { - envs, err := impl.collectObjectStoreCertificates( - ctx, - types.NamespacedName{ - Name: pluginConfiguration.RecoveryBarmanObjectName, - Namespace: namespace, - }, - ) - if err != nil { - return nil, err - } - result = append(result, envs...) - } - return result, nil } From ef7afbbc2986d762c92ecc8d0bea29b97ba5436c Mon Sep 17 00:00:00 2001 From: Leonardo Cecchi Date: Fri, 9 May 2025 16:17:31 +0200 Subject: [PATCH 10/12] chore: review docs Signed-off-by: Leonardo Cecchi --- web/docs/usage.md | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/web/docs/usage.md b/web/docs/usage.md index a8bdd729..0eef7871 100644 --- a/web/docs/usage.md +++ b/web/docs/usage.md @@ -214,7 +214,9 @@ spec: ## Configuring the plugin instance sidecar The Barman Cloud Plugin uses a sidecar container that runs alongside each -PostgreSQL instance pod. This sidecar handles backup, WAL archiving, and restore +PostgreSQL instance pod. + +This sidecar handles backup, WAL archiving, and restore operations. You can control how the sidecar works by setting the `.spec.instanceSidecarConfiguration` section in your `ObjectStore` resource. These settings apply to all PostgreSQL instances that use this object store. @@ -238,10 +240,27 @@ spec: cpu: "500m" ``` -When the plugin is enabled, the sidecar is automatically injected into each -PostgreSQL instance pod. Even if you define multiple `ObjectStore` resources, -only one sidecar will run per instance. If a replica cluster also archives WALs -to a different `ObjectStore`, the sidecar will use the resource settings from the -`ObjectStore` referenced by the archiving plugin, not the one in the -`.spec.externalClusters` section. Changes to the sidecar configuration will be -applied on the next reconciliation of the `Cluster` resources it. +The plugin injects a sidecar in the recovery job and in the PostgreSQL +instance Pods when needed. + +When this happens, the sidecar will manage multiple `ObjectStore` +resources: + +1. the target object store, where WAL files and backups will + be written + +2. the replica source object store, used by the log-shipping designated + primary to get the WAL files + +3. the recovery object store, used when creating the cluster from an + existing backup (used only by the recovery job) + +The resources defined by the recovery object store will be used when +injecting the sidecar in the recovery job. + +If a sidecar is needed by PG instances pod, the resources defined in +the target object store will be used. Should this object store be not +defined, the replica object store will be used. + +Changes to the sidecar configuration will be applied on the next +reconciliation of the `Cluster` resources it. From 74916768dd753a3627d446ec2cc4ead25e572ccd Mon Sep 17 00:00:00 2001 From: Francesco Canovai Date: Fri, 9 May 2025 17:10:59 +0200 Subject: [PATCH 11/12] docs: improve Signed-off-by: Francesco Canovai --- web/docs/usage.md | 64 +++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/web/docs/usage.md b/web/docs/usage.md index 0eef7871..869dfe29 100644 --- a/web/docs/usage.md +++ b/web/docs/usage.md @@ -213,13 +213,25 @@ spec: ## Configuring the plugin instance sidecar -The Barman Cloud Plugin uses a sidecar container that runs alongside each -PostgreSQL instance pod. - -This sidecar handles backup, WAL archiving, and restore -operations. You can control how the sidecar works by setting the -`.spec.instanceSidecarConfiguration` section in your `ObjectStore` resource. -These settings apply to all PostgreSQL instances that use this object store. +The Barman Cloud Plugin runs as a sidecar container next to each PostgreSQL +instance pod. It manages backup, WAL archiving, and restore processes. + +Configuration comes from multiple `ObjectStore` resources: + +1. The one referenced in the + `.spec.plugins` section of the `Cluster`. This is the + object store used for WAL archiving and base backups. +2. The one referenced in the external cluster + used in the `.spec.replica.source` section of the `Cluster`. This is + used by the log-shipping designated primary to get the WAL files. +3. The one referenced in the + `.spec.bootstrap.recovery.source` section of the `Cluster`. Used by + the initial recovery job to create the cluster from an existing backup. + +You can fine-tune sidecar behavior in the `.spec.instanceSidecarConfiguration` +of your ObjectStore. These settings apply to all PostgreSQL instances that use +this object store. Any updates take effect at the next `Cluster` reconciliation, +and could generate a rollout of the `Cluster`. ```yaml apiVersion: barmancloud.cnpg.io/v1 @@ -228,39 +240,19 @@ metadata: name: minio-store spec: configuration: - # [...] + # [...] instanceSidecarConfiguration: retentionPolicyIntervalSeconds: 1800 resources: requests: - memory: "64Mi" - cpu: "250m" + memory: "XXX" + cpu: "YYY" limits: - memory: "512Mi" - cpu: "500m" + memory: "XXX" + cpu: "YYY" ``` -The plugin injects a sidecar in the recovery job and in the PostgreSQL -instance Pods when needed. - -When this happens, the sidecar will manage multiple `ObjectStore` -resources: - -1. the target object store, where WAL files and backups will - be written - -2. the replica source object store, used by the log-shipping designated - primary to get the WAL files - -3. the recovery object store, used when creating the cluster from an - existing backup (used only by the recovery job) - -The resources defined by the recovery object store will be used when -injecting the sidecar in the recovery job. - -If a sidecar is needed by PG instances pod, the resources defined in -the target object store will be used. Should this object store be not -defined, the replica object store will be used. - -Changes to the sidecar configuration will be applied on the next -reconciliation of the `Cluster` resources it. +:::note +If more than one `ObjectStore` applies, the `instanceSidecarConfiguration` of +the one set in `.spec.plugins` has priority. +::: From 42de3e68562d00bac59ec778c2de1b997e82cc89 Mon Sep 17 00:00:00 2001 From: Francesco Canovai Date: Fri, 9 May 2025 17:16:01 +0200 Subject: [PATCH 12/12] docs: spellcheck wordlist Signed-off-by: Francesco Canovai --- .wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.wordlist.txt b/.wordlist.txt index 0f035f0d..f520ed04 100644 --- a/.wordlist.txt +++ b/.wordlist.txt @@ -104,6 +104,7 @@ repos retentionCheckInterval retentionPolicy rolebinding +rollout sc secretKeyRef selfsigned