From 3c8691d76d58a901e72f5d3eb2a79ea724775c27 Mon Sep 17 00:00:00 2001 From: qicz Date: Mon, 26 Jun 2023 20:30:17 +0800 Subject: [PATCH 01/14] fix: envoy proxy resource apply bug. Signed-off-by: qicz --- internal/infrastructure/kubernetes/infra_resource.go | 3 +++ internal/infrastructure/kubernetes/proxy/resource.go | 3 ++- .../infrastructure/kubernetes/proxy/resource_provider.go | 2 ++ .../kubernetes/proxy/resource_provider_test.go | 3 ++- .../kubernetes/proxy/testdata/deployments/bootstrap.yaml | 3 +++ .../proxy/testdata/deployments/component-level.yaml | 3 +++ .../kubernetes/proxy/testdata/deployments/custom.yaml | 3 +++ .../kubernetes/proxy/testdata/deployments/default-env.yaml | 3 +++ .../kubernetes/proxy/testdata/deployments/default.yaml | 3 +++ .../proxy/testdata/deployments/extension-env.yaml | 3 +++ .../kubernetes/proxy/testdata/deployments/volumes.yaml | 3 +++ internal/infrastructure/kubernetes/ratelimit/resource.go | 7 +++++-- .../kubernetes/ratelimit/resource_provider.go | 2 ++ .../kubernetes/ratelimit/resource_provider_test.go | 3 ++- .../ratelimit/testdata/deployments/affinity.yaml | 3 +++ .../kubernetes/ratelimit/testdata/deployments/custom.yaml | 3 +++ .../ratelimit/testdata/deployments/default-env.yaml | 3 +++ .../kubernetes/ratelimit/testdata/deployments/default.yaml | 3 +++ .../ratelimit/testdata/deployments/extension-env.yaml | 3 +++ .../ratelimit/testdata/deployments/override-env.yaml | 3 +++ .../ratelimit/testdata/deployments/redis-tls-settings.yaml | 4 ++++ .../ratelimit/testdata/deployments/tolerations.yaml | 4 ++++ .../kubernetes/ratelimit/testdata/deployments/volumes.yaml | 4 ++++ 23 files changed, 69 insertions(+), 5 deletions(-) diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index e2b1b1b396..237902936a 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -73,6 +73,9 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) } return i.Client.CreateOrUpdate(ctx, key, current, deployment, func() bool { + deployment.Spec.Template.Spec.DeprecatedServiceAccount = current.Spec.Template.Spec.DeprecatedServiceAccount + deployment.Spec.Template.Spec.SecurityContext = current.Spec.Template.Spec.SecurityContext + deployment.Spec.Replicas = current.Spec.Replicas return !reflect.DeepEqual(deployment.Spec, current.Spec) }) } diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 7e6574eb75..de08032ab2 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -217,7 +217,8 @@ func expectedDeploymentVolumes(name string, deploymentSpec *egcfgv1a1.Kubernetes Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "envoy", + SecretName: "envoy", + DefaultMode: pointer.Int32(int32(420)), }, }, }, diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider.go b/internal/infrastructure/kubernetes/proxy/resource_provider.go index 8d74ce9d14..702b5d7e94 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider.go @@ -203,6 +203,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Volumes: expectedDeploymentVolumes(r.infra.Name, deploymentConfig), }, }, + RevisionHistoryLimit: pointer.Int32(10), + ProgressDeadlineSeconds: pointer.Int32(600), }, } diff --git a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go index 5bc9e58d5a..e8d17482ac 100644 --- a/internal/infrastructure/kubernetes/proxy/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/proxy/resource_provider_test.go @@ -204,7 +204,8 @@ func TestDeployment(t *testing.T) { Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "custom-envoy-cert", + SecretName: "custom-envoy-cert", + DefaultMode: pointer.Int32(420), }, }, }, diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml index 73e86a058a..8778473b33 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml @@ -84,6 +84,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -94,3 +95,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml index b5a5d5cd48..418e9d5858 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml @@ -85,6 +85,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -95,3 +96,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml index f9fa2b7fb7..da35636ee5 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml @@ -190,6 +190,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -200,3 +201,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml index f9fa2b7fb7..da35636ee5 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml @@ -190,6 +190,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -200,3 +201,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml index f9ece0c355..2e21b94411 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml @@ -181,6 +181,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -191,3 +192,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml index c74059582b..10e49b68f8 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml @@ -194,6 +194,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -204,3 +205,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml index 56ec3c19b0..0b3b642b35 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml @@ -194,6 +194,7 @@ spec: - name: certs secret: secretName: custom-envoy-cert + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -204,3 +205,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/resource.go b/internal/infrastructure/kubernetes/ratelimit/resource.go index f35de689d4..6e52e48a52 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource.go @@ -12,6 +12,7 @@ import ( "strconv" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" @@ -161,7 +162,8 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme Name: "redis-certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: string(rateLimit.Backend.Redis.TLS.CertificateRef.Name), + SecretName: string(rateLimit.Backend.Redis.TLS.CertificateRef.Name), + DefaultMode: pointer.Int32(int32(420)), }, }, }) @@ -171,7 +173,8 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "envoy-rate-limit", + SecretName: "envoy-rate-limit", + DefaultMode: pointer.Int32(int32(420)), }, }, }) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 4b9d29a3df..cf10db6638 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -133,6 +133,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Tolerations: r.rateLimitDeployment.Pod.Tolerations, }, }, + RevisionHistoryLimit: pointer.Int32(10), + ProgressDeadlineSeconds: pointer.Int32(600), }, } diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index 78f8d1a8ab..60094e8df3 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -438,7 +438,8 @@ func TestDeployment(t *testing.T) { Name: "certs", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: "custom-cert", + SecretName: "custom-cert", + DefaultMode: pointer.Int32(420), }, }, }, diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml index d8d74e7ccd..38e10279a7 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml @@ -110,3 +110,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml index 62ce1df16a..c222e32184 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml @@ -101,3 +101,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml index 62ce1df16a..c222e32184 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml @@ -101,3 +101,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml index a988eb88f3..b3c7471fe5 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default.yaml @@ -92,3 +92,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml index cab16f4d72..eb3b46367a 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml @@ -105,3 +105,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml index 0160876c07..e217646575 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml @@ -101,3 +101,6 @@ spec: - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml index 45a2ac4920..c4c44bdb50 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml @@ -112,6 +112,10 @@ spec: - name: redis-certs secret: secretName: ratelimit-cert + defaultMode: 420 - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml index 55ee463638..319621cd2c 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml @@ -117,6 +117,10 @@ spec: - name: redis-certs secret: secretName: ratelimit-cert + defaultMode: 420 - name: certs secret: secretName: envoy-rate-limit + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml index a4e285cc9f..28f582a305 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml @@ -117,6 +117,10 @@ spec: - name: redis-certs secret: secretName: ratelimit-cert-origin + defaultMode: 420 - name: certs secret: secretName: custom-cert + defaultMode: 420 + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 From fdfa5ef044864a03c1d8eaf813172d28897cb782 Mon Sep 17 00:00:00 2001 From: qicz Date: Mon, 26 Jun 2023 23:47:59 +0800 Subject: [PATCH 02/14] update pointer. Signed-off-by: qicz --- internal/infrastructure/kubernetes/proxy/resource.go | 4 ++-- internal/infrastructure/kubernetes/ratelimit/resource.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index de08032ab2..2f0773897c 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -218,7 +218,7 @@ func expectedDeploymentVolumes(name string, deploymentSpec *egcfgv1a1.Kubernetes VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "envoy", - DefaultMode: pointer.Int32(int32(420)), + DefaultMode: pointer.Int32(420), }, }, }, @@ -239,7 +239,7 @@ func expectedDeploymentVolumes(name string, deploymentSpec *egcfgv1a1.Kubernetes Path: SdsCertFilename, }, }, - DefaultMode: pointer.Int32(int32(420)), + DefaultMode: pointer.Int32(420), Optional: pointer.Bool(false), }, }, diff --git a/internal/infrastructure/kubernetes/ratelimit/resource.go b/internal/infrastructure/kubernetes/ratelimit/resource.go index 6e52e48a52..e7a39f8383 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource.go @@ -163,7 +163,7 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: string(rateLimit.Backend.Redis.TLS.CertificateRef.Name), - DefaultMode: pointer.Int32(int32(420)), + DefaultMode: pointer.Int32(420), }, }, }) @@ -174,7 +174,7 @@ func expectedDeploymentVolumes(rateLimit *egcfgv1a1.RateLimit, rateLimitDeployme VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "envoy-rate-limit", - DefaultMode: pointer.Int32(int32(420)), + DefaultMode: pointer.Int32(420), }, }, }) From 2887058e7cbc5026afc23bfb4378264b2b40e20a Mon Sep 17 00:00:00 2001 From: qicz Date: Mon, 26 Jun 2023 23:52:19 +0800 Subject: [PATCH 03/14] add comment Signed-off-by: qicz --- internal/infrastructure/kubernetes/infra_resource.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 237902936a..809fc13457 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -73,8 +73,11 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) } return i.Client.CreateOrUpdate(ctx, key, current, deployment, func() bool { + // applied to k8s the "DeprecatedServiceAccount" will fill it. deployment.Spec.Template.Spec.DeprecatedServiceAccount = current.Spec.Template.Spec.DeprecatedServiceAccount + // applied to k8s the "SecurityContext" will fill it with default settings. deployment.Spec.Template.Spec.SecurityContext = current.Spec.Template.Spec.SecurityContext + // adapter the hpa updating. deployment.Spec.Replicas = current.Spec.Replicas return !reflect.DeepEqual(deployment.Spec, current.Spec) }) From 328a7dc95624c4dc7b58e8172e1c665bd5b53635 Mon Sep 17 00:00:00 2001 From: qicz Date: Tue, 27 Jun 2023 00:10:57 +0800 Subject: [PATCH 04/14] update cm cmp logic. Signed-off-by: qicz --- .../infrastructure/kubernetes/infra_resource.go | 2 +- .../infrastructure/kubernetes/proxy_infra_test.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 809fc13457..23f2236e23 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -54,7 +54,7 @@ func (i *Infra) createOrUpdateConfigMap(ctx context.Context, r ResourceRender) e } return i.Client.CreateOrUpdate(ctx, key, current, cm, func() bool { - return !reflect.DeepEqual(cm.Data, current.Data) + return !reflect.DeepEqual(cm.Data, current.Data) || !reflect.DeepEqual(cm.BinaryData, current.BinaryData) }) } diff --git a/internal/infrastructure/kubernetes/proxy_infra_test.go b/internal/infrastructure/kubernetes/proxy_infra_test.go index 7328b10706..61e32749e1 100644 --- a/internal/infrastructure/kubernetes/proxy_infra_test.go +++ b/internal/infrastructure/kubernetes/proxy_infra_test.go @@ -7,8 +7,10 @@ package kubernetes import ( "context" + "reflect" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -30,6 +32,18 @@ func newTestInfra(t *testing.T) *Infra { return newTestInfraWithClient(t, cli) } +func TestCmpBytes(t *testing.T) { + m1 := map[string][]byte{} + m1["a"] = []byte("aaa") + m2 := map[string][]byte{} + m2["a"] = []byte("aaa") + + assert.True(t, reflect.DeepEqual(m1, m2)) + assert.False(t, reflect.DeepEqual(nil, m2)) + assert.False(t, reflect.DeepEqual(m1, nil)) + assert.True(t, reflect.DeepEqual(nil, nil)) +} + func newTestInfraWithClient(t *testing.T, cli client.Client) *Infra { cfg, err := config.New() require.NoError(t, err) From 9a3eb1ee4092968627c08511599f6fa371401d84 Mon Sep 17 00:00:00 2001 From: qicz Date: Tue, 27 Jun 2023 08:24:55 +0800 Subject: [PATCH 05/14] fix lint Signed-off-by: qicz --- internal/infrastructure/kubernetes/proxy_infra_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/infrastructure/kubernetes/proxy_infra_test.go b/internal/infrastructure/kubernetes/proxy_infra_test.go index 61e32749e1..d3180439d6 100644 --- a/internal/infrastructure/kubernetes/proxy_infra_test.go +++ b/internal/infrastructure/kubernetes/proxy_infra_test.go @@ -41,7 +41,6 @@ func TestCmpBytes(t *testing.T) { assert.True(t, reflect.DeepEqual(m1, m2)) assert.False(t, reflect.DeepEqual(nil, m2)) assert.False(t, reflect.DeepEqual(m1, nil)) - assert.True(t, reflect.DeepEqual(nil, nil)) } func newTestInfraWithClient(t *testing.T, cli client.Client) *Infra { From 7ed2fe7787ab6904080b7a25dfdbe0f9577a2b29 Mon Sep 17 00:00:00 2001 From: qicz Date: Tue, 27 Jun 2023 14:27:04 +0800 Subject: [PATCH 06/14] add probe field default value. Signed-off-by: qicz --- internal/infrastructure/kubernetes/proxy/resource.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 2f0773897c..6ea48dc782 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -158,10 +158,14 @@ func expectedProxyContainers(infra *ir.ProxyInfra, deploymentConfig *egcfgv1a1.K ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ - Path: bootstrap.EnvoyReadinessPath, - Port: intstr.IntOrString{Type: intstr.Int, IntVal: bootstrap.EnvoyReadinessPort}, + Path: bootstrap.EnvoyReadinessPath, + Port: intstr.IntOrString{Type: intstr.Int, IntVal: bootstrap.EnvoyReadinessPort}, + Scheme: corev1.URISchemeHTTP, }, }, + TimeoutSeconds: 1, + SuccessThreshold: 1, + FailureThreshold: 3, }, }, } From 2c933e3d5cdb983f14250dbd53a7fe2bb5861fdf Mon Sep 17 00:00:00 2001 From: qicz Date: Tue, 27 Jun 2023 14:29:58 +0800 Subject: [PATCH 07/14] fix uts Signed-off-by: qicz --- .../kubernetes/proxy/testdata/deployments/bootstrap.yaml | 4 ++++ .../proxy/testdata/deployments/component-level.yaml | 4 ++++ .../kubernetes/proxy/testdata/deployments/custom.yaml | 4 ++++ .../kubernetes/proxy/testdata/deployments/default-env.yaml | 4 ++++ .../kubernetes/proxy/testdata/deployments/default.yaml | 4 ++++ .../kubernetes/proxy/testdata/deployments/extension-env.yaml | 4 ++++ .../kubernetes/proxy/testdata/deployments/volumes.yaml | 4 ++++ 7 files changed, 28 insertions(+) diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml index 8778473b33..73dce26517 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml @@ -67,6 +67,10 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml index 418e9d5858..426dd59b82 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml @@ -68,6 +68,10 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml index da35636ee5..97a8c1d11d 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml @@ -169,6 +169,10 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml index da35636ee5..97a8c1d11d 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml @@ -169,6 +169,10 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml index 2e21b94411..9c3881c492 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml @@ -164,6 +164,10 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml index 10e49b68f8..64006c05c0 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml @@ -173,6 +173,10 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml index 0b3b642b35..a42abd10ff 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml @@ -173,6 +173,10 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File securityContext: From abba9dd527403afb35e65b1151464a2b87caf660 Mon Sep 17 00:00:00 2001 From: qicz Date: Wed, 28 Jun 2023 18:37:01 +0800 Subject: [PATCH 08/14] align probe Signed-off-by: qicz --- internal/infrastructure/kubernetes/infra_resource.go | 6 ++++-- internal/infrastructure/kubernetes/proxy/resource.go | 1 + .../kubernetes/proxy/testdata/deployments/bootstrap.yaml | 1 + .../proxy/testdata/deployments/component-level.yaml | 1 + .../kubernetes/proxy/testdata/deployments/custom.yaml | 1 + .../kubernetes/proxy/testdata/deployments/default-env.yaml | 1 + .../kubernetes/proxy/testdata/deployments/default.yaml | 1 + .../proxy/testdata/deployments/extension-env.yaml | 1 + .../kubernetes/proxy/testdata/deployments/volumes.yaml | 1 + 9 files changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 23f2236e23..8e94970e9c 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -77,8 +77,10 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) deployment.Spec.Template.Spec.DeprecatedServiceAccount = current.Spec.Template.Spec.DeprecatedServiceAccount // applied to k8s the "SecurityContext" will fill it with default settings. deployment.Spec.Template.Spec.SecurityContext = current.Spec.Template.Spec.SecurityContext - // adapter the hpa updating. - deployment.Spec.Replicas = current.Spec.Replicas + // adapter the hpa updating and envoyproxy updating. + if *deployment.Spec.Replicas < *current.Spec.Replicas { + deployment.Spec.Replicas = current.Spec.Replicas + } return !reflect.DeepEqual(deployment.Spec, current.Spec) }) } diff --git a/internal/infrastructure/kubernetes/proxy/resource.go b/internal/infrastructure/kubernetes/proxy/resource.go index 6ea48dc782..da14cc9d1d 100644 --- a/internal/infrastructure/kubernetes/proxy/resource.go +++ b/internal/infrastructure/kubernetes/proxy/resource.go @@ -164,6 +164,7 @@ func expectedProxyContainers(infra *ir.ProxyInfra, deploymentConfig *egcfgv1a1.K }, }, TimeoutSeconds: 1, + PeriodSeconds: 10, SuccessThreshold: 1, FailureThreshold: 3, }, diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml index 73dce26517..ae80e44166 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/bootstrap.yaml @@ -69,6 +69,7 @@ spec: port: 19001 scheme: HTTP timeoutSeconds: 1 + periodSeconds: 10 successThreshold: 1 failureThreshold: 3 terminationMessagePath: /dev/termination-log diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml index 426dd59b82..2a28fd490a 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/component-level.yaml @@ -70,6 +70,7 @@ spec: port: 19001 scheme: HTTP timeoutSeconds: 1 + periodSeconds: 10 successThreshold: 1 failureThreshold: 3 terminationMessagePath: /dev/termination-log diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml index 97a8c1d11d..052aab28b8 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/custom.yaml @@ -171,6 +171,7 @@ spec: port: 19001 scheme: HTTP timeoutSeconds: 1 + periodSeconds: 10 successThreshold: 1 failureThreshold: 3 terminationMessagePath: /dev/termination-log diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml index 97a8c1d11d..052aab28b8 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default-env.yaml @@ -171,6 +171,7 @@ spec: port: 19001 scheme: HTTP timeoutSeconds: 1 + periodSeconds: 10 successThreshold: 1 failureThreshold: 3 terminationMessagePath: /dev/termination-log diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml index 9c3881c492..a5fc99d981 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/default.yaml @@ -166,6 +166,7 @@ spec: port: 19001 scheme: HTTP timeoutSeconds: 1 + periodSeconds: 10 successThreshold: 1 failureThreshold: 3 terminationMessagePath: /dev/termination-log diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml index 64006c05c0..154a601650 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/extension-env.yaml @@ -175,6 +175,7 @@ spec: port: 19001 scheme: HTTP timeoutSeconds: 1 + periodSeconds: 10 successThreshold: 1 failureThreshold: 3 terminationMessagePath: /dev/termination-log diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml index a42abd10ff..904dde7412 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/volumes.yaml @@ -175,6 +175,7 @@ spec: port: 19001 scheme: HTTP timeoutSeconds: 1 + periodSeconds: 10 successThreshold: 1 failureThreshold: 3 terminationMessagePath: /dev/termination-log From 7af91378ee5e68dca25611ab06b80e30990f5f0b Mon Sep 17 00:00:00 2001 From: qicz Date: Fri, 30 Jun 2023 11:49:49 +0800 Subject: [PATCH 09/14] optimize deploy compare logic Signed-off-by: qicz --- .../kubernetes/infra_resource.go | 10 +--------- .../kubernetes/resource/resource.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index 8e94970e9c..47b8d22481 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -73,15 +73,7 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) } return i.Client.CreateOrUpdate(ctx, key, current, deployment, func() bool { - // applied to k8s the "DeprecatedServiceAccount" will fill it. - deployment.Spec.Template.Spec.DeprecatedServiceAccount = current.Spec.Template.Spec.DeprecatedServiceAccount - // applied to k8s the "SecurityContext" will fill it with default settings. - deployment.Spec.Template.Spec.SecurityContext = current.Spec.Template.Spec.SecurityContext - // adapter the hpa updating and envoyproxy updating. - if *deployment.Spec.Replicas < *current.Spec.Replicas { - deployment.Spec.Replicas = current.Spec.Replicas - } - return !reflect.DeepEqual(deployment.Spec, current.Spec) + return !resource.CompareDeployment(current, deployment) }) } diff --git a/internal/infrastructure/kubernetes/resource/resource.go b/internal/infrastructure/kubernetes/resource/resource.go index 96fbf2a5b4..8109390559 100644 --- a/internal/infrastructure/kubernetes/resource/resource.go +++ b/internal/infrastructure/kubernetes/resource/resource.go @@ -6,8 +6,11 @@ package resource import ( + "reflect" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -41,6 +44,21 @@ func CompareSvc(currentSvc, originalSvc *corev1.Service) bool { cmpopts.IgnoreFields(corev1.ServiceSpec{}, "ClusterIP", "ClusterIPs")) } +// CompareDeployment compare the current from the k8s and deployment from the resource_provider. +func CompareDeployment(current, deployment *appv1.Deployment) bool { + // applied to k8s the "DeprecatedServiceAccount" will fill it. + deployment.Spec.Template.Spec.DeprecatedServiceAccount = current.Spec.Template.Spec.DeprecatedServiceAccount + // applied to k8s the "SecurityContext" will fill it with default settings. + if deployment.Spec.Template.Spec.SecurityContext == nil { + deployment.Spec.Template.Spec.SecurityContext = current.Spec.Template.Spec.SecurityContext + } + // adapter the hpa updating and envoyproxy updating. + if *deployment.Spec.Replicas < *current.Spec.Replicas { + deployment.Spec.Replicas = current.Spec.Replicas + } + return reflect.DeepEqual(deployment.Spec, current.Spec) +} + // ExpectedProxyContainerEnv returns expected container envs. func ExpectedProxyContainerEnv(container *egcfgv1a1.KubernetesContainerSpec, env []corev1.EnvVar) []corev1.EnvVar { amendFunc := func(envVar corev1.EnvVar) { From fa4c2cd8a51470f40ade27c0ad58ec3cfa25ce05 Mon Sep 17 00:00:00 2001 From: qicz Date: Fri, 30 Jun 2023 14:05:53 +0800 Subject: [PATCH 10/14] add compare deploy uts Signed-off-by: qicz --- .../kubernetes/resource/resource.go | 3 + .../kubernetes/resource/resource_test.go | 100 ++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/internal/infrastructure/kubernetes/resource/resource.go b/internal/infrastructure/kubernetes/resource/resource.go index 8109390559..a11b99f4f1 100644 --- a/internal/infrastructure/kubernetes/resource/resource.go +++ b/internal/infrastructure/kubernetes/resource/resource.go @@ -48,14 +48,17 @@ func CompareSvc(currentSvc, originalSvc *corev1.Service) bool { func CompareDeployment(current, deployment *appv1.Deployment) bool { // applied to k8s the "DeprecatedServiceAccount" will fill it. deployment.Spec.Template.Spec.DeprecatedServiceAccount = current.Spec.Template.Spec.DeprecatedServiceAccount + // applied to k8s the "SecurityContext" will fill it with default settings. if deployment.Spec.Template.Spec.SecurityContext == nil { deployment.Spec.Template.Spec.SecurityContext = current.Spec.Template.Spec.SecurityContext } + // adapter the hpa updating and envoyproxy updating. if *deployment.Spec.Replicas < *current.Spec.Replicas { deployment.Spec.Replicas = current.Spec.Replicas } + return reflect.DeepEqual(deployment.Spec, current.Spec) } diff --git a/internal/infrastructure/kubernetes/resource/resource_test.go b/internal/infrastructure/kubernetes/resource/resource_test.go index 314fa4e650..91a1b5883e 100644 --- a/internal/infrastructure/kubernetes/resource/resource_test.go +++ b/internal/infrastructure/kubernetes/resource/resource_test.go @@ -8,8 +8,10 @@ package resource import ( "testing" + appv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -436,3 +438,101 @@ func TestExpectedContainerVolumeMounts(t *testing.T) { }) } } + +func TestCompareDeployment(t *testing.T) { + type args struct { + current *appv1.Deployment + deployment *appv1.Deployment + } + tests := []struct { + name string + args args + notEqual bool + }{ + { + name: "not eq DeprecatedServiceAccount", + args: args{ + current: &appv1.Deployment{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: appv1.DeploymentSpec{ + Replicas: pointer.Int32(2), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + DeprecatedServiceAccount: "a", + }, + }, + }, + Status: appv1.DeploymentStatus{}, + }, + deployment: &appv1.Deployment{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: appv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + DeprecatedServiceAccount: "b", + }, + }, + }, + Status: appv1.DeploymentStatus{}, + }, + }, + notEqual: true, + }, + { + name: "not eq SecurityContext", + args: args{ + current: &appv1.Deployment{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: appv1.DeploymentSpec{ + Replicas: pointer.Int32(2), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + DeprecatedServiceAccount: "a", + SecurityContext: &corev1.PodSecurityContext{ + SELinuxOptions: nil, + WindowsOptions: nil, + RunAsUser: nil, + RunAsGroup: nil, + RunAsNonRoot: nil, + SupplementalGroups: nil, + FSGroup: nil, + Sysctls: nil, + FSGroupChangePolicy: nil, + SeccompProfile: nil, + }, + }, + }, + }, + Status: appv1.DeploymentStatus{}, + }, + deployment: &appv1.Deployment{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: appv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{ + DeprecatedServiceAccount: "b", + }, + }, + }, + Status: appv1.DeploymentStatus{}, + }, + }, + notEqual: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.notEqual, CompareDeployment(tt.args.current, tt.args.deployment), "CompareDeployment(%v, %v)", tt.args.current, tt.args.deployment) + }) + } +} From c53e9a93e2d93c3e1907a4a8db991c761185fba2 Mon Sep 17 00:00:00 2001 From: qicz Date: Tue, 25 Jul 2023 16:05:57 +0800 Subject: [PATCH 11/14] rm cm binarydata cmp Signed-off-by: qicz --- internal/infrastructure/kubernetes/infra_resource.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index f62d41be57..c8c8e80937 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -56,7 +56,7 @@ func (i *Infra) createOrUpdateConfigMap(ctx context.Context, r ResourceRender) e } return i.Client.CreateOrUpdate(ctx, key, current, cm, func() bool { - return !reflect.DeepEqual(cm.Data, current.Data) || !reflect.DeepEqual(cm.BinaryData, current.BinaryData) + return !reflect.DeepEqual(cm.Data, current.Data) }) } From 3a181301b733b8a668f7014e293b8149fef0602f Mon Sep 17 00:00:00 2001 From: qicz Date: Wed, 26 Jul 2023 17:26:01 +0800 Subject: [PATCH 12/14] rm deploy cmp logic Signed-off-by: qicz --- .../kubernetes/infra_resource.go | 2 +- .../kubernetes/resource/resource.go | 21 ---- .../kubernetes/resource/resource_test.go | 105 +----------------- 3 files changed, 3 insertions(+), 125 deletions(-) diff --git a/internal/infrastructure/kubernetes/infra_resource.go b/internal/infrastructure/kubernetes/infra_resource.go index c8c8e80937..af041ad331 100644 --- a/internal/infrastructure/kubernetes/infra_resource.go +++ b/internal/infrastructure/kubernetes/infra_resource.go @@ -75,7 +75,7 @@ func (i *Infra) createOrUpdateDeployment(ctx context.Context, r ResourceRender) } return i.Client.CreateOrUpdate(ctx, key, current, deployment, func() bool { - return !resource.CompareDeployment(current, deployment) + return !reflect.DeepEqual(deployment.Spec, current.Spec) }) } diff --git a/internal/infrastructure/kubernetes/resource/resource.go b/internal/infrastructure/kubernetes/resource/resource.go index a11b99f4f1..96fbf2a5b4 100644 --- a/internal/infrastructure/kubernetes/resource/resource.go +++ b/internal/infrastructure/kubernetes/resource/resource.go @@ -6,11 +6,8 @@ package resource import ( - "reflect" - "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - appv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,24 +41,6 @@ func CompareSvc(currentSvc, originalSvc *corev1.Service) bool { cmpopts.IgnoreFields(corev1.ServiceSpec{}, "ClusterIP", "ClusterIPs")) } -// CompareDeployment compare the current from the k8s and deployment from the resource_provider. -func CompareDeployment(current, deployment *appv1.Deployment) bool { - // applied to k8s the "DeprecatedServiceAccount" will fill it. - deployment.Spec.Template.Spec.DeprecatedServiceAccount = current.Spec.Template.Spec.DeprecatedServiceAccount - - // applied to k8s the "SecurityContext" will fill it with default settings. - if deployment.Spec.Template.Spec.SecurityContext == nil { - deployment.Spec.Template.Spec.SecurityContext = current.Spec.Template.Spec.SecurityContext - } - - // adapter the hpa updating and envoyproxy updating. - if *deployment.Spec.Replicas < *current.Spec.Replicas { - deployment.Spec.Replicas = current.Spec.Replicas - } - - return reflect.DeepEqual(deployment.Spec, current.Spec) -} - // ExpectedProxyContainerEnv returns expected container envs. func ExpectedProxyContainerEnv(container *egcfgv1a1.KubernetesContainerSpec, env []corev1.EnvVar) []corev1.EnvVar { amendFunc := func(envVar corev1.EnvVar) { diff --git a/internal/infrastructure/kubernetes/resource/resource_test.go b/internal/infrastructure/kubernetes/resource/resource_test.go index 91a1b5883e..4673096085 100644 --- a/internal/infrastructure/kubernetes/resource/resource_test.go +++ b/internal/infrastructure/kubernetes/resource/resource_test.go @@ -8,14 +8,11 @@ package resource import ( "testing" - appv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" egcfgv1a1 "github.com/envoyproxy/gateway/api/config/v1alpha1" ) @@ -438,101 +435,3 @@ func TestExpectedContainerVolumeMounts(t *testing.T) { }) } } - -func TestCompareDeployment(t *testing.T) { - type args struct { - current *appv1.Deployment - deployment *appv1.Deployment - } - tests := []struct { - name string - args args - notEqual bool - }{ - { - name: "not eq DeprecatedServiceAccount", - args: args{ - current: &appv1.Deployment{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: appv1.DeploymentSpec{ - Replicas: pointer.Int32(2), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{}, - Spec: corev1.PodSpec{ - DeprecatedServiceAccount: "a", - }, - }, - }, - Status: appv1.DeploymentStatus{}, - }, - deployment: &appv1.Deployment{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: appv1.DeploymentSpec{ - Replicas: pointer.Int32(1), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{}, - Spec: corev1.PodSpec{ - DeprecatedServiceAccount: "b", - }, - }, - }, - Status: appv1.DeploymentStatus{}, - }, - }, - notEqual: true, - }, - { - name: "not eq SecurityContext", - args: args{ - current: &appv1.Deployment{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: appv1.DeploymentSpec{ - Replicas: pointer.Int32(2), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{}, - Spec: corev1.PodSpec{ - DeprecatedServiceAccount: "a", - SecurityContext: &corev1.PodSecurityContext{ - SELinuxOptions: nil, - WindowsOptions: nil, - RunAsUser: nil, - RunAsGroup: nil, - RunAsNonRoot: nil, - SupplementalGroups: nil, - FSGroup: nil, - Sysctls: nil, - FSGroupChangePolicy: nil, - SeccompProfile: nil, - }, - }, - }, - }, - Status: appv1.DeploymentStatus{}, - }, - deployment: &appv1.Deployment{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{}, - Spec: appv1.DeploymentSpec{ - Replicas: pointer.Int32(1), - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{}, - Spec: corev1.PodSpec{ - DeprecatedServiceAccount: "b", - }, - }, - }, - Status: appv1.DeploymentStatus{}, - }, - }, - notEqual: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equalf(t, tt.notEqual, CompareDeployment(tt.args.current, tt.args.deployment), "CompareDeployment(%v, %v)", tt.args.current, tt.args.deployment) - }) - } -} From 8a70ff091c3b51c7e44b33a41fcbb5a319cf7b58 Mon Sep 17 00:00:00 2001 From: qicz Date: Wed, 26 Jul 2023 18:16:14 +0800 Subject: [PATCH 13/14] fix ut Signed-off-by: qicz --- .../proxy/testdata/deployments/enable-prometheus.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml b/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml index 51b6103bc3..e100ed3e76 100644 --- a/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml +++ b/internal/infrastructure/kubernetes/proxy/testdata/deployments/enable-prometheus.yaml @@ -190,6 +190,11 @@ spec: httpGet: path: /ready port: 19001 + scheme: HTTP + timeoutSeconds: 1 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 3 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: @@ -207,6 +212,7 @@ spec: - name: certs secret: secretName: envoy + defaultMode: 420 - configMap: defaultMode: 420 items: @@ -217,3 +223,5 @@ spec: name: envoy-default-64656661 optional: false name: sds + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 From ff25ed3cc5ef348e1a64ce8a035fe6f0262c0785 Mon Sep 17 00:00:00 2001 From: qicz Date: Thu, 27 Jul 2023 08:23:35 +0800 Subject: [PATCH 14/14] fix lint Signed-off-by: qicz --- .../testdata/deployments/redis-tls-settings.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml index 3d9b300a68..dacf0988f5 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml @@ -114,13 +114,13 @@ spec: runAsUser: 1000 terminationGracePeriodSeconds: 300 volumes: - - name: redis-certs - secret: - secretName: ratelimit-cert - defaultMode: 420 - - name: certs - secret: - secretName: envoy-rate-limit - defaultMode: 420 + - name: redis-certs + secret: + secretName: ratelimit-cert + defaultMode: 420 + - name: certs + secret: + secretName: envoy-rate-limit + defaultMode: 420 revisionHistoryLimit: 10 progressDeadlineSeconds: 600