From e4dc874407347bd5762978f708132a9ec6fa6308 Mon Sep 17 00:00:00 2001 From: vizeit Date: Fri, 19 Jan 2024 13:46:59 -0800 Subject: [PATCH 1/5] 36888 Template field support for configmaps --- .../providers/cncf/kubernetes/operators/pod.py | 10 +++++++--- .../cncf/kubernetes/operators/test_pod.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 8a701b675a19d..dc61deaf979e3 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -225,6 +225,7 @@ class KubernetesPodOperator(BaseOperator): "volumes", "volume_mounts", "cluster_context", + "configmaps", ) template_fields_renderers = {"env_vars": "py"} @@ -306,8 +307,9 @@ def __init__( if pod_runtime_info_envs: self.env_vars.extend([convert_pod_runtime_info_env(p) for p in pod_runtime_info_envs]) self.env_from = env_from or [] - if configmaps: - self.env_from.extend([convert_configmap(c) for c in configmaps]) + self.configmaps = configmaps + if self.configmaps: + self.env_from.extend([convert_configmap(c) for c in self.configmaps]) self.ports = [convert_port(p) for p in ports] if ports else [] self.volume_mounts = [convert_volume_mount(v) for v in volume_mounts] if volume_mounts else [] self.volumes = [convert_volume(volume) for volume in volumes] if volumes else [] @@ -403,11 +405,13 @@ def _render_nested_template_fields( elif isinstance(content, k8s.V1ResourceRequirements): template_fields = ("limits", "requests") elif isinstance(content, k8s.V1Volume): - template_fields = ("name", "persistent_volume_claim") + template_fields = ("name", "persistent_volume_claim", "config_map") elif isinstance(content, k8s.V1VolumeMount): template_fields = ("name", "sub_path") elif isinstance(content, k8s.V1PersistentVolumeClaimVolumeSource): template_fields = ("claim_name",) + elif isinstance(content, k8s.V1ConfigMapVolumeSource): + template_fields = ("name",) else: template_fields = None diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index 690dece9534c9..1de4a2c735556 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -151,6 +151,18 @@ def test_templates(self, create_task_instance_of_operator): cmds="{{ dag.dag_id }}", image="{{ dag.dag_id }}", annotations={"dag-id": "{{ dag.dag_id }}"}, + configmaps=["{{ dag.dag_id }}"], + volumes=[ + k8s.V1Volume( + name="{{ dag.dag_id }}", + config_map=k8s.V1ConfigMapVolumeSource( + name="{{ dag.dag_id }}", + items=[ + k8s.V1KeyToPath(key="key", path='path') + ] + ) + ) + ], ) rendered = ti.render_templates() @@ -170,6 +182,9 @@ def test_templates(self, create_task_instance_of_operator): assert dag_id == ti.task.arguments assert dag_id == ti.task.env_vars[0] assert dag_id == rendered.annotations["dag-id"] + assert dag_id == ti.task.configmaps[0] + assert dag_id == rendered.volumes[0].name + assert dag_id == rendered.volumes[0].config_map.name def run_pod(self, operator: KubernetesPodOperator, map_index: int = -1) -> k8s.V1Pod: with self.dag_maker(dag_id="dag") as dag: From 853af48b3f2b0d75cc2171d986719bed1051e0c8 Mon Sep 17 00:00:00 2001 From: vizeit Date: Fri, 19 Jan 2024 20:47:02 -0800 Subject: [PATCH 2/5] 36888 Add newsfragment - correct format --- newsfragments/36922.feature.rst | 1 + tests/providers/cncf/kubernetes/operators/test_pod.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 newsfragments/36922.feature.rst diff --git a/newsfragments/36922.feature.rst b/newsfragments/36922.feature.rst new file mode 100644 index 0000000000000..8c23ca0239aa6 --- /dev/null +++ b/newsfragments/36922.feature.rst @@ -0,0 +1 @@ +Support for configmaps in the template fields in the KubernetesPodOperator diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index 1de4a2c735556..4c40ccbacfb59 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -156,11 +156,8 @@ def test_templates(self, create_task_instance_of_operator): k8s.V1Volume( name="{{ dag.dag_id }}", config_map=k8s.V1ConfigMapVolumeSource( - name="{{ dag.dag_id }}", - items=[ - k8s.V1KeyToPath(key="key", path='path') - ] - ) + name="{{ dag.dag_id }}", items=[k8s.V1KeyToPath(key="key", path="path")] + ), ) ], ) From 1b33aba9dce542a70648af5e75ef305c9f3aff9b Mon Sep 17 00:00:00 2001 From: vizeit Date: Fri, 19 Jan 2024 13:46:59 -0800 Subject: [PATCH 3/5] 36888 Template field support for configmaps --- .../providers/cncf/kubernetes/operators/pod.py | 10 +++++++--- .../cncf/kubernetes/operators/test_pod.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/operators/pod.py b/airflow/providers/cncf/kubernetes/operators/pod.py index 8a701b675a19d..dc61deaf979e3 100644 --- a/airflow/providers/cncf/kubernetes/operators/pod.py +++ b/airflow/providers/cncf/kubernetes/operators/pod.py @@ -225,6 +225,7 @@ class KubernetesPodOperator(BaseOperator): "volumes", "volume_mounts", "cluster_context", + "configmaps", ) template_fields_renderers = {"env_vars": "py"} @@ -306,8 +307,9 @@ def __init__( if pod_runtime_info_envs: self.env_vars.extend([convert_pod_runtime_info_env(p) for p in pod_runtime_info_envs]) self.env_from = env_from or [] - if configmaps: - self.env_from.extend([convert_configmap(c) for c in configmaps]) + self.configmaps = configmaps + if self.configmaps: + self.env_from.extend([convert_configmap(c) for c in self.configmaps]) self.ports = [convert_port(p) for p in ports] if ports else [] self.volume_mounts = [convert_volume_mount(v) for v in volume_mounts] if volume_mounts else [] self.volumes = [convert_volume(volume) for volume in volumes] if volumes else [] @@ -403,11 +405,13 @@ def _render_nested_template_fields( elif isinstance(content, k8s.V1ResourceRequirements): template_fields = ("limits", "requests") elif isinstance(content, k8s.V1Volume): - template_fields = ("name", "persistent_volume_claim") + template_fields = ("name", "persistent_volume_claim", "config_map") elif isinstance(content, k8s.V1VolumeMount): template_fields = ("name", "sub_path") elif isinstance(content, k8s.V1PersistentVolumeClaimVolumeSource): template_fields = ("claim_name",) + elif isinstance(content, k8s.V1ConfigMapVolumeSource): + template_fields = ("name",) else: template_fields = None diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index 690dece9534c9..1de4a2c735556 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -151,6 +151,18 @@ def test_templates(self, create_task_instance_of_operator): cmds="{{ dag.dag_id }}", image="{{ dag.dag_id }}", annotations={"dag-id": "{{ dag.dag_id }}"}, + configmaps=["{{ dag.dag_id }}"], + volumes=[ + k8s.V1Volume( + name="{{ dag.dag_id }}", + config_map=k8s.V1ConfigMapVolumeSource( + name="{{ dag.dag_id }}", + items=[ + k8s.V1KeyToPath(key="key", path='path') + ] + ) + ) + ], ) rendered = ti.render_templates() @@ -170,6 +182,9 @@ def test_templates(self, create_task_instance_of_operator): assert dag_id == ti.task.arguments assert dag_id == ti.task.env_vars[0] assert dag_id == rendered.annotations["dag-id"] + assert dag_id == ti.task.configmaps[0] + assert dag_id == rendered.volumes[0].name + assert dag_id == rendered.volumes[0].config_map.name def run_pod(self, operator: KubernetesPodOperator, map_index: int = -1) -> k8s.V1Pod: with self.dag_maker(dag_id="dag") as dag: From bea6d5e0593c4fe50837c40041125e7db8e422c8 Mon Sep 17 00:00:00 2001 From: vizeit Date: Fri, 19 Jan 2024 20:47:02 -0800 Subject: [PATCH 4/5] 36888 Add newsfragment - correct format --- newsfragments/36922.feature.rst | 1 + tests/providers/cncf/kubernetes/operators/test_pod.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) create mode 100644 newsfragments/36922.feature.rst diff --git a/newsfragments/36922.feature.rst b/newsfragments/36922.feature.rst new file mode 100644 index 0000000000000..8c23ca0239aa6 --- /dev/null +++ b/newsfragments/36922.feature.rst @@ -0,0 +1 @@ +Support for configmaps in the template fields in the KubernetesPodOperator diff --git a/tests/providers/cncf/kubernetes/operators/test_pod.py b/tests/providers/cncf/kubernetes/operators/test_pod.py index 1de4a2c735556..4c40ccbacfb59 100644 --- a/tests/providers/cncf/kubernetes/operators/test_pod.py +++ b/tests/providers/cncf/kubernetes/operators/test_pod.py @@ -156,11 +156,8 @@ def test_templates(self, create_task_instance_of_operator): k8s.V1Volume( name="{{ dag.dag_id }}", config_map=k8s.V1ConfigMapVolumeSource( - name="{{ dag.dag_id }}", - items=[ - k8s.V1KeyToPath(key="key", path='path') - ] - ) + name="{{ dag.dag_id }}", items=[k8s.V1KeyToPath(key="key", path="path")] + ), ) ], ) From cec5ad4a036218d76932b9a2f9594edd83c2939a Mon Sep 17 00:00:00 2001 From: vizeit Date: Fri, 19 Jan 2024 23:54:43 -0800 Subject: [PATCH 5/5] Remove newsfragment --- newsfragments/36922.feature.rst | 1 - 1 file changed, 1 deletion(-) delete mode 100644 newsfragments/36922.feature.rst diff --git a/newsfragments/36922.feature.rst b/newsfragments/36922.feature.rst deleted file mode 100644 index 8c23ca0239aa6..0000000000000 --- a/newsfragments/36922.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Support for configmaps in the template fields in the KubernetesPodOperator