From d4a4a988eda7fac43811f991aac97cb619d4c25e Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Mon, 15 Apr 2024 13:28:49 +0100 Subject: [PATCH 01/17] gitSync: add proxySettingsSecret to support when repository behind corporate proxy --- chart/templates/_helpers.yaml | 17 +++++++++++++++++ chart/values.schema.json | 8 ++++++++ chart/values.yaml | 1 + 3 files changed, 26 insertions(+) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 7606e75084a31..5fe01e536931a 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -245,6 +245,23 @@ If release name contains chart name it will be used as a full name. name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GITSYNC_PASSWORD {{- end }} + {{ if .Values.dags.gitSync.proxySettingsSecret }} + - name: HTTPS_PROXY + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.proxySettingsSecret | quote }} + key: HTTPS_PROXY + - name: HTTP_PROXY + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.proxySettingsSecret | quote }} + key: HTTP_PROXY + - name: NO_PROXY + valueFrom: + secretKeyRef: + name: {{ .Values.dags.gitSync.proxySettingsSecret | quote }} + key: NO_PROXY + {{- end }} - name: GIT_SYNC_REV value: {{ .Values.dags.gitSync.rev | quote }} - name: GITSYNC_REF diff --git a/chart/values.schema.json b/chart/values.schema.json index de298ec0620de..49397ba990b11 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -8481,6 +8481,14 @@ ], "default": null }, + "proxySettingsSecret": { + "description": "The name of the Secret containing the proxy settings (HTTPS_PROXY, HTTP_PROXY, and NO_PROXY) when the repository is behind a proxy.", + "type": [ + "string", + "null" + ], + "default": null + }, "sshKeySecret": { "description": "Name of a Secret containing the repo `sshKeySecret`.", "type": [ diff --git a/chart/values.yaml b/chart/values.yaml index 4fe4f05756566..55905aaec56d0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2621,6 +2621,7 @@ dags: # # credentialsSecret: git-credentials # + # proxySettingsSecret: proxy-settings # # If you are using an ssh clone url, you can load # the ssh private key to a k8s secret like the one below From 67a8ec8f5b49069667af9cf15f6e83e14cd206f0 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Mon, 15 Apr 2024 16:08:21 +0100 Subject: [PATCH 02/17] gitSync: upd proxySettingsSecret add comments in values.yaml --- chart/templates/_helpers.yaml | 2 +- chart/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 5fe01e536931a..f44c73a1967d0 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -245,7 +245,7 @@ If release name contains chart name it will be used as a full name. name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GITSYNC_PASSWORD {{- end }} - {{ if .Values.dags.gitSync.proxySettingsSecret }} + {{- if .Values.dags.gitSync.proxySettingsSecret }} - name: HTTPS_PROXY valueFrom: secretKeyRef: diff --git a/chart/values.yaml b/chart/values.yaml index 55905aaec56d0..a15db577b01cf 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2621,6 +2621,7 @@ dags: # # credentialsSecret: git-credentials # + # name of the secret containing proxy settings for git-sync: HTTPS_PROXY, HTTP_PROXY, NO_PROXY # proxySettingsSecret: proxy-settings # # If you are using an ssh clone url, you can load From f1b5ccfb45b47c805899f9b399706866c58ba084 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Mon, 15 Apr 2024 16:11:34 +0100 Subject: [PATCH 03/17] gitSync: proxySettingsSecret upd comments in values.yaml --- chart/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chart/values.yaml b/chart/values.yaml index a15db577b01cf..ff71f94fea840 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2621,7 +2621,8 @@ dags: # # credentialsSecret: git-credentials # - # name of the secret containing proxy settings for git-sync: HTTPS_PROXY, HTTP_PROXY, NO_PROXY + # Name of the secret containing proxy settings for git-sync. + # The secret must include keys for HTTPS_PROXY, HTTP_PROXY, and NO_PROXY. # proxySettingsSecret: proxy-settings # # If you are using an ssh clone url, you can load From b326ca27ba555e9d8523486e857b085e87bd4164 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Tue, 16 Apr 2024 00:07:40 +0100 Subject: [PATCH 04/17] add envFrom to git_sync_container template --- chart/templates/_helpers.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index f44c73a1967d0..313be77e737de 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -198,6 +198,7 @@ If release name contains chart name it will be used as a full name. image: {{ template "git_sync_image" . }} imagePullPolicy: {{ .Values.images.gitSync.pullPolicy }} securityContext: {{- include "localContainerSecurityContext" .Values.dags.gitSync | nindent 4 }} + envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 2 }} env: {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey }} - name: GIT_SSH_KEY_FILE From 6aed02e3988c2dfd486609443a28b187055fdf34 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Tue, 16 Apr 2024 00:10:23 +0100 Subject: [PATCH 05/17] Revert "gitSync: proxySettingsSecret upd comments in values.yaml" This reverts commit f38fb21a0c1373d389b7c5d979a75c912196d183. --- chart/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index ff71f94fea840..a15db577b01cf 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2621,8 +2621,7 @@ dags: # # credentialsSecret: git-credentials # - # Name of the secret containing proxy settings for git-sync. - # The secret must include keys for HTTPS_PROXY, HTTP_PROXY, and NO_PROXY. + # name of the secret containing proxy settings for git-sync: HTTPS_PROXY, HTTP_PROXY, NO_PROXY # proxySettingsSecret: proxy-settings # # If you are using an ssh clone url, you can load From 6729da116544238b6f514cf0f2be049249a7dccc Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Tue, 16 Apr 2024 00:10:43 +0100 Subject: [PATCH 06/17] Revert "gitSync: upd proxySettingsSecret add comments in values.yaml" This reverts commit d25fc2f7585edac96d40cada6217328cbdfd3e18. --- chart/templates/_helpers.yaml | 2 +- chart/values.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 313be77e737de..64cb74930fb13 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -246,7 +246,7 @@ If release name contains chart name it will be used as a full name. name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GITSYNC_PASSWORD {{- end }} - {{- if .Values.dags.gitSync.proxySettingsSecret }} + {{ if .Values.dags.gitSync.proxySettingsSecret }} - name: HTTPS_PROXY valueFrom: secretKeyRef: diff --git a/chart/values.yaml b/chart/values.yaml index a15db577b01cf..55905aaec56d0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2621,7 +2621,6 @@ dags: # # credentialsSecret: git-credentials # - # name of the secret containing proxy settings for git-sync: HTTPS_PROXY, HTTP_PROXY, NO_PROXY # proxySettingsSecret: proxy-settings # # If you are using an ssh clone url, you can load From a5422d10aea94c8918180a4bd99f03d67d6b6357 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Tue, 16 Apr 2024 00:11:03 +0100 Subject: [PATCH 07/17] Revert "gitSync: add proxySettingsSecret to support when repository behind corporate proxy" This reverts commit 905991b1be7c85ee968d2d4d113b2d1ebbb6aaaa. --- chart/templates/_helpers.yaml | 17 ----------------- chart/values.yaml | 1 - 2 files changed, 18 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 64cb74930fb13..9414c4a7ddba1 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -246,23 +246,6 @@ If release name contains chart name it will be used as a full name. name: {{ .Values.dags.gitSync.credentialsSecret | quote }} key: GITSYNC_PASSWORD {{- end }} - {{ if .Values.dags.gitSync.proxySettingsSecret }} - - name: HTTPS_PROXY - valueFrom: - secretKeyRef: - name: {{ .Values.dags.gitSync.proxySettingsSecret | quote }} - key: HTTPS_PROXY - - name: HTTP_PROXY - valueFrom: - secretKeyRef: - name: {{ .Values.dags.gitSync.proxySettingsSecret | quote }} - key: HTTP_PROXY - - name: NO_PROXY - valueFrom: - secretKeyRef: - name: {{ .Values.dags.gitSync.proxySettingsSecret | quote }} - key: NO_PROXY - {{- end }} - name: GIT_SYNC_REV value: {{ .Values.dags.gitSync.rev | quote }} - name: GITSYNC_REF diff --git a/chart/values.yaml b/chart/values.yaml index 55905aaec56d0..4fe4f05756566 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -2621,7 +2621,6 @@ dags: # # credentialsSecret: git-credentials # - # proxySettingsSecret: proxy-settings # # If you are using an ssh clone url, you can load # the ssh private key to a k8s secret like the one below From b0f033a21c33bfda40527adf6910df6a69e38ed0 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Wed, 1 May 2024 20:08:09 +0300 Subject: [PATCH 08/17] fix tests to validate changes --- helm_tests/airflow_aux/test_pod_template_file.py | 1 + helm_tests/other/test_git_sync_scheduler.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 43a8d1942343a..5ed6f1b2dacbd 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -98,6 +98,7 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", + "envFrom": [], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index 02289bd2deffc..6976f5512fb7f 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -89,6 +89,7 @@ def test_validate_the_git_sync_container_spec(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", + "envFrom": [], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, @@ -149,6 +150,7 @@ def test_validate_the_git_sync_container_spec_if_wait_specified(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", + "envFrom": [], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, From 0dc8d266346bc2ad1056b7e2f1f3875588bf8232 Mon Sep 17 00:00:00 2001 From: rkarimov Date: Mon, 24 Jun 2024 17:53:50 +0300 Subject: [PATCH 09/17] test extraEnvFrom --- helm_tests/airflow_aux/test_pod_template_file.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 5ed6f1b2dacbd..80827080fc8a7 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -87,6 +87,13 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "knownHosts": None, } }, + "extraEnvFrom": [ + { + "secretRef": { + "name": "airflow-proxy-config" + } + } + ], }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, From 49e9efd9e7fb60efba3ac775a15c9f4d55bf52aa Mon Sep 17 00:00:00 2001 From: rkarimov Date: Mon, 24 Jun 2024 18:24:44 +0300 Subject: [PATCH 10/17] test extraEnvFrom --- chart/values.yaml | 5 +++++ helm_tests/airflow_aux/test_pod_template_file.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index 4fe4f05756566..6afb2537cc199 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -342,6 +342,11 @@ extraSecrets: {} # '{{ .Release.Name }}-other-secret-name-suffix': # data: | # ... +# 'proxy-config': +# stringData: | +# HTTP_PROXY: http://proxyuser:proxy_password@192.168.0.10:2080 +# HTTPS_PROXY: http://proxyuser:proxy_password@192.168.0.10:2080 +# NO_PROXY: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc" # Extra ConfigMaps that will be managed by the chart # (You can use them with extraEnv or extraEnvFrom or some of the extraVolumes values). diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 80827080fc8a7..4f6b567ab24bc 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -90,7 +90,7 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "extraEnvFrom": [ { "secretRef": { - "name": "airflow-proxy-config" + "name": "proxy-config" } } ], @@ -105,7 +105,13 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", - "envFrom": [], + "envFrom": [ + { + "secretRef": { + "name": "proxy-config" + } + } + ], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, From dfdd18470024eb4d6c83a0b4f4cd507ab6297b4d Mon Sep 17 00:00:00 2001 From: rkarimov Date: Mon, 24 Jun 2024 19:50:57 +0300 Subject: [PATCH 11/17] upd extraEnvFrom --- chart/values.yaml | 4 ++-- helm_tests/airflow_aux/test_pod_template_file.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/chart/values.yaml b/chart/values.yaml index 6afb2537cc199..0a0136af7abbf 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -344,8 +344,8 @@ extraSecrets: {} # ... # 'proxy-config': # stringData: | -# HTTP_PROXY: http://proxyuser:proxy_password@192.168.0.10:2080 -# HTTPS_PROXY: http://proxyuser:proxy_password@192.168.0.10:2080 +# HTTP_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 +# HTTPS_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 # NO_PROXY: "localhost,127.0.0.1,.svc.cluster.local,kubernetes.default.svc" # Extra ConfigMaps that will be managed by the chart diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 4f6b567ab24bc..b4fc60710b7b7 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -89,9 +89,9 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): }, "extraEnvFrom": [ { - "secretRef": { - "name": "proxy-config" - } + "secretRef": { + "name": "proxy-config" + } } ], }, From 6af3c5288d30de3a359123b67cb18a85e53df098 Mon Sep 17 00:00:00 2001 From: rkarimov Date: Mon, 24 Jun 2024 20:18:49 +0300 Subject: [PATCH 12/17] test_git_sync_scheduler.py upd extraEnvFrom --- helm_tests/other/test_git_sync_scheduler.py | 30 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index 6976f5512fb7f..fa9c67a0bb242 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -80,6 +80,13 @@ def test_validate_the_git_sync_container_spec(self): }, "persistence": {"enabled": True}, }, + "extraEnvFrom": [ + { + "secretRef": { + "name": "proxy-config" + } + } + ], }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) @@ -89,7 +96,13 @@ def test_validate_the_git_sync_container_spec(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", - "envFrom": [], + "envFrom": [ + { + "secretRef": { + "name": "proxy-config" + } + } + ], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, @@ -141,6 +154,13 @@ def test_validate_the_git_sync_container_spec_if_wait_specified(self): }, "persistence": {"enabled": True}, }, + "extraEnvFrom": [ + { + "secretRef": { + "name": "proxy-config" + } + } + ], }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) @@ -150,7 +170,13 @@ def test_validate_the_git_sync_container_spec_if_wait_specified(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", - "envFrom": [], + "envFrom": [ + { + "secretRef": { + "name": "proxy-config" + } + } + ], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, From 13308242d275debd0771061298f1c79e117a75a5 Mon Sep 17 00:00:00 2001 From: rkarimov Date: Tue, 25 Jun 2024 00:56:13 +0300 Subject: [PATCH 13/17] fix schema.json --- chart/values.schema.json | 3 +- .../airflow_aux/test_pod_template_file.py | 16 ++-------- helm_tests/other/test_git_sync_scheduler.py | 32 +++---------------- 3 files changed, 8 insertions(+), 43 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index 49397ba990b11..28257162bf2dd 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -950,7 +950,8 @@ "description": "Extra envFrom 'items' that will be added to the definition of Airflow containers; a string is expected (can be templated).", "type": [ "null", - "string" + "string", + "array" ], "x-docsSection": "Airflow", "default": null, diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index b4fc60710b7b7..8284d8efbaf83 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -87,13 +87,7 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "knownHosts": None, } }, - "extraEnvFrom": [ - { - "secretRef": { - "name": "proxy-config" - } - } - ], + "extraEnvFrom": [{"secretRef": {"name": "proxy-config"}}], }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -105,13 +99,7 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", - "envFrom": [ - { - "secretRef": { - "name": "proxy-config" - } - } - ], + "envFrom": [{"secretRef": {"name": "proxy-config"}}], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index fa9c67a0bb242..f24f8cd8e7741 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -80,13 +80,7 @@ def test_validate_the_git_sync_container_spec(self): }, "persistence": {"enabled": True}, }, - "extraEnvFrom": [ - { - "secretRef": { - "name": "proxy-config" - } - } - ], + "extraEnvFrom": [{"secretRef": {"name": "proxy-config"}}], }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) @@ -96,13 +90,7 @@ def test_validate_the_git_sync_container_spec(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", - "envFrom": [ - { - "secretRef": { - "name": "proxy-config" - } - } - ], + "envFrom": [{"secretRef": {"name": "proxy-config"}}], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, @@ -154,13 +142,7 @@ def test_validate_the_git_sync_container_spec_if_wait_specified(self): }, "persistence": {"enabled": True}, }, - "extraEnvFrom": [ - { - "secretRef": { - "name": "proxy-config" - } - } - ], + "extraEnvFrom": [{"secretRef": {"name": "proxy-config"}}], }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) @@ -170,13 +152,7 @@ def test_validate_the_git_sync_container_spec_if_wait_specified(self): "securityContext": {"runAsUser": 65533}, "image": "test-registry/test-repo:test-tag", "imagePullPolicy": "Always", - "envFrom": [ - { - "secretRef": { - "name": "proxy-config" - } - } - ], + "envFrom": [{"secretRef": {"name": "proxy-config"}}], "env": [ {"name": "GIT_SYNC_REV", "value": "HEAD"}, {"name": "GITSYNC_REF", "value": "test-branch"}, From a11035ebd052504ee0bc7b71b785471ed16c2da9 Mon Sep 17 00:00:00 2001 From: Rasul Karimov Date: Wed, 3 Jul 2024 13:37:55 +0500 Subject: [PATCH 14/17] gitSync define a separate dags.gitSync.envFrom for gitSync containers --- chart/templates/_helpers.yaml | 10 +++++++++- chart/values.schema.json | 13 +++++++++++++ chart/values.yaml | 7 ++++++- helm_tests/airflow_aux/test_pod_template_file.py | 2 +- helm_tests/other/test_git_sync_scheduler.py | 4 ++-- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 9414c4a7ddba1..93802bd723b3f 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -184,6 +184,14 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{/* User defined gitSync container environment from */}} +{{- define "custom_git_sync_environment_from" }} + {{- $Global := . }} + {{- with .Values.dags.gitSync.envFrom }} + {{- tpl . $Global | nindent 2 }} + {{- end }} +{{- end }} + {{/* Git ssh key volume */}} {{- define "git_sync_ssh_key_volume" }} - name: git-sync-ssh-key @@ -198,7 +206,7 @@ If release name contains chart name it will be used as a full name. image: {{ template "git_sync_image" . }} imagePullPolicy: {{ .Values.images.gitSync.pullPolicy }} securityContext: {{- include "localContainerSecurityContext" .Values.dags.gitSync | nindent 4 }} - envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 2 }} + envFrom: {{- include "custom_git_sync_environment_from" . | default "\n []" | indent 2 }} env: {{- if or .Values.dags.gitSync.sshKeySecret .Values.dags.gitSync.sshKey }} - name: GIT_SSH_KEY_FILE diff --git a/chart/values.schema.json b/chart/values.schema.json index 28257162bf2dd..aa677c7acbd39 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -8537,6 +8537,19 @@ } ] }, + "envFrom": { + "description": "Extra envFrom 'items' that will be added to the definition of Airflow gitSync containers; a string or array are expected (can be templated).", + "type": [ + "null", + "string", + "array" + ], + "default": null, + "examples": [ + "- secretRef:\n name: 'proxy-configs", + "- configMapRef:\n name: 'proxy-configs" + ] + }, "resources": { "description": "Resources on workers git-sync sidecar", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 0a0136af7abbf..2d7741b1530b7 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -342,7 +342,7 @@ extraSecrets: {} # '{{ .Release.Name }}-other-secret-name-suffix': # data: | # ... -# 'proxy-config': +# 'proxy-configs': # stringData: | # HTTP_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 # HTTPS_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 @@ -2660,6 +2660,11 @@ dags: # For backwards compatibility, wait will be used if it is specified. period: 5s wait: ~ + # add variables from secret into gitSync containers, such proxy-configs + envFrom: ~ + # envFrom: | + # - secretRef: + # name: 'proxy-configs' containerName: git-sync uid: 65533 diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 8284d8efbaf83..79fe738805afb 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -85,9 +85,9 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "sshKeySecret": None, "credentialsSecret": None, "knownHosts": None, + "envFrom": [{"secretRef": {"name": "proxy-config"}}], } }, - "extraEnvFrom": [{"secretRef": {"name": "proxy-config"}}], }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index f24f8cd8e7741..8cc7afaa4039f 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -77,10 +77,10 @@ def test_validate_the_git_sync_container_spec(self): "sshKeySecret": None, "credentialsSecret": None, "knownHosts": None, + "envFrom": [{"secretRef": {"name": "proxy-config"}}], }, "persistence": {"enabled": True}, }, - "extraEnvFrom": [{"secretRef": {"name": "proxy-config"}}], }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) @@ -139,10 +139,10 @@ def test_validate_the_git_sync_container_spec_if_wait_specified(self): "sshKeySecret": None, "credentialsSecret": None, "knownHosts": None, + "envFrom": [{"secretRef": {"name": "proxy-config"}}], }, "persistence": {"enabled": True}, }, - "extraEnvFrom": [{"secretRef": {"name": "proxy-config"}}], }, show_only=["templates/scheduler/scheduler-deployment.yaml"], ) From 6efb7f6d20b606c2b26abf9aa4c0703a16f95343 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Wed, 3 Jul 2024 14:20:26 +0500 Subject: [PATCH 15/17] gitSync remove proxySettingsSecret --- chart/values.schema.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index aa677c7acbd39..6f08d12c7b83d 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -8482,14 +8482,6 @@ ], "default": null }, - "proxySettingsSecret": { - "description": "The name of the Secret containing the proxy settings (HTTPS_PROXY, HTTP_PROXY, and NO_PROXY) when the repository is behind a proxy.", - "type": [ - "string", - "null" - ], - "default": null - }, "sshKeySecret": { "description": "Name of a Secret containing the repo `sshKeySecret`.", "type": [ From 949eb2ed95f4d04742448140a914e92290c74239 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Wed, 3 Jul 2024 14:23:38 +0500 Subject: [PATCH 16/17] gitSync secret name proxy-config --- chart/values.schema.json | 4 ++-- chart/values.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index 6f08d12c7b83d..44b9e13e8066a 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -8538,8 +8538,8 @@ ], "default": null, "examples": [ - "- secretRef:\n name: 'proxy-configs", - "- configMapRef:\n name: 'proxy-configs" + "- secretRef:\n name: 'proxy-config", + "- configMapRef:\n name: 'proxy-config" ] }, "resources": { diff --git a/chart/values.yaml b/chart/values.yaml index 2d7741b1530b7..76130e55bb5a0 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -342,7 +342,7 @@ extraSecrets: {} # '{{ .Release.Name }}-other-secret-name-suffix': # data: | # ... -# 'proxy-configs': +# 'proxy-config': # stringData: | # HTTP_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 # HTTPS_PROXY: http://proxy_user:proxy_password@192.168.0.10:2080 @@ -2660,11 +2660,11 @@ dags: # For backwards compatibility, wait will be used if it is specified. period: 5s wait: ~ - # add variables from secret into gitSync containers, such proxy-configs + # add variables from secret into gitSync containers, such proxy-config envFrom: ~ # envFrom: | # - secretRef: - # name: 'proxy-configs' + # name: 'proxy-config' containerName: git-sync uid: 65533 From b304be43729eb05f0d959c6b8773ef6382e477d1 Mon Sep 17 00:00:00 2001 From: rasulkarimov Date: Wed, 3 Jul 2024 15:58:24 +0500 Subject: [PATCH 17/17] gitSync: fix helm tests --- chart/values.schema.json | 6 ++---- helm_tests/airflow_aux/test_pod_template_file.py | 2 +- helm_tests/other/test_git_sync_scheduler.py | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/chart/values.schema.json b/chart/values.schema.json index 44b9e13e8066a..5f6a3b55d89f5 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -950,8 +950,7 @@ "description": "Extra envFrom 'items' that will be added to the definition of Airflow containers; a string is expected (can be templated).", "type": [ "null", - "string", - "array" + "string" ], "x-docsSection": "Airflow", "default": null, @@ -8533,8 +8532,7 @@ "description": "Extra envFrom 'items' that will be added to the definition of Airflow gitSync containers; a string or array are expected (can be templated).", "type": [ "null", - "string", - "array" + "string" ], "default": null, "examples": [ diff --git a/helm_tests/airflow_aux/test_pod_template_file.py b/helm_tests/airflow_aux/test_pod_template_file.py index 79fe738805afb..8d4cabeef0497 100644 --- a/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm_tests/airflow_aux/test_pod_template_file.py @@ -85,7 +85,7 @@ def test_should_add_an_init_container_if_git_sync_is_true(self): "sshKeySecret": None, "credentialsSecret": None, "knownHosts": None, - "envFrom": [{"secretRef": {"name": "proxy-config"}}], + "envFrom": "- secretRef:\n name: 'proxy-config'\n", } }, }, diff --git a/helm_tests/other/test_git_sync_scheduler.py b/helm_tests/other/test_git_sync_scheduler.py index 8cc7afaa4039f..4610888a3779d 100644 --- a/helm_tests/other/test_git_sync_scheduler.py +++ b/helm_tests/other/test_git_sync_scheduler.py @@ -77,7 +77,7 @@ def test_validate_the_git_sync_container_spec(self): "sshKeySecret": None, "credentialsSecret": None, "knownHosts": None, - "envFrom": [{"secretRef": {"name": "proxy-config"}}], + "envFrom": "- secretRef:\n name: 'proxy-config'\n", }, "persistence": {"enabled": True}, }, @@ -139,7 +139,7 @@ def test_validate_the_git_sync_container_spec_if_wait_specified(self): "sshKeySecret": None, "credentialsSecret": None, "knownHosts": None, - "envFrom": [{"secretRef": {"name": "proxy-config"}}], + "envFrom": "- secretRef:\n name: 'proxy-config'\n", }, "persistence": {"enabled": True}, },