diff --git a/chart/values.schema.json b/chart/values.schema.json index e9270a5943b4c..5035bfa7d1e39 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -2688,8 +2688,11 @@ "properties": { "replicas": { "description": "Number of Airflow Celery workers.", - "type": "integer", - "default": 1 + "type": [ + "integer", + "null" + ], + "default": null }, "revisionHistoryLimit": { "description": "Max number of old Airflow Celery workers ReplicaSets to retain.", @@ -2720,11 +2723,7 @@ "items": { "type": "string" }, - "default": [ - "bash", - "-c", - "exec \\\nairflow {{ semverCompare \">=2.0.0\" .Values.airflowVersion | ternary \"celery worker\" \"worker\" }}\n{{- if and .Values.workers.queue (ne .Values.workers.queue \"default\") }}\n{{- \" -q \" }}{{ .Values.workers.queue }}\n{{- end }}" - ] + "default": null }, "livenessProbe": { "description": "Liveness probe configuration for Airflow Celery worker containers.", @@ -2733,28 +2732,43 @@ "properties": { "enabled": { "description": "Enable liveness probe for Airflow Celery workers.", - "type": "boolean", - "default": true + "type": [ + "boolean", + "null" + ], + "default": null }, "initialDelaySeconds": { "description": "Number of seconds after the container has started before liveness probes are initiated.", - "type": "integer", - "default": 10 + "type": [ + "integer", + "null" + ], + "default": null }, "timeoutSeconds": { "description": "Number of seconds after which the probe times out. Minimum value is 1 seconds.", - "type": "integer", - "default": 20 + "type": [ + "integer", + "null" + ], + "default": null }, "failureThreshold": { "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Minimum value is 1.", - "type": "integer", - "default": 5 + "type": [ + "integer", + "null" + ], + "default": null }, "periodSeconds": { "description": "How often (in seconds) to perform the probe. Minimum value is 1.", - "type": "integer", - "default": 60 + "type": [ + "integer", + "null" + ], + "default": null }, "command": { "description": "Command for livenessProbe", @@ -2782,12 +2796,7 @@ "null", "object" ], - "default": { - "rollingUpdate": { - "maxSurge": "100%", - "maxUnavailable": "50%" - } - } + "default": null }, "podManagementPolicy": { "description": "Specifies the policy for managing pods within the Airflow Celery worker. Only applicable to StatefulSet.", @@ -2863,8 +2872,11 @@ "properties": { "enabled": { "description": "Enable persistent volumes.", - "type": "boolean", - "default": true + "type": [ + "boolean", + "null" + ], + "default": null }, "persistentVolumeClaimRetentionPolicy": { "$ref": "#/definitions/persistentVolumeClaimRetentionPolicy", @@ -2872,8 +2884,11 @@ }, "size": { "description": "Volume size for Airflow Celery worker StatefulSet.", - "type": "string", - "default": "100Gi" + "type": [ + "string", + "null" + ], + "default": null }, "storageClassName": { "description": "If using a custom StorageClass, pass name ref to all StatefulSets here (templated).", @@ -2885,8 +2900,11 @@ }, "fixPermissions": { "description": "Execute init container to chown log directory. This is currently only needed in kind, due to usage of local-path provisioner.", - "type": "boolean", - "default": false + "type": [ + "boolean", + "null" + ], + "default": null }, "annotations": { "description": "Annotations to add to Airflow Celery worker volumes.", @@ -2929,8 +2947,11 @@ "properties": { "enabled": { "description": "Enable Kerberos init container.", - "type": "boolean", - "default": false + "type": [ + "boolean", + "null" + ], + "default": null }, "resources": { "description": "Resources on kerberos init container.", @@ -3066,8 +3087,11 @@ "properties": { "enabled": { "description": "Enable kerberos init container.", - "type": "boolean", - "default": false + "type": [ + "boolean", + "null" + ], + "default": null }, "resources": { "description": "Resources on kerberos init container.", diff --git a/chart/values.yaml b/chart/values.yaml index 0197f4eb5fa75..224c9f650bffb 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1041,7 +1041,7 @@ workers: celery: # Number of Airflow Celery workers - replicas: 1 + replicas: ~ # Max number of old Airflow Celery workers ReplicaSets to retain revisionHistoryLimit: ~ @@ -1050,24 +1050,16 @@ workers: command: ~ # Args to use when running Airflow Celery workers (templated) - args: - - "bash" - - "-c" - - |- - exec \ - airflow {{ semverCompare ">=2.0.0" .Values.airflowVersion | ternary "celery worker" "worker" }} - {{- if and .Values.workers.queue (ne .Values.workers.queue "default") }} - {{- " -q " }}{{ .Values.workers.queue }} - {{- end }} + args: ~ # If the Airflow Celery worker stops responding for 5 minutes (5*60s) # kill the worker and let Kubernetes restart it livenessProbe: - enabled: true - initialDelaySeconds: 10 - timeoutSeconds: 20 - failureThreshold: 5 - periodSeconds: 60 + enabled: ~ + initialDelaySeconds: ~ + timeoutSeconds: ~ + failureThreshold: ~ + periodSeconds: ~ command: ~ # Enable the default workers defined by the root `workers` and `workers.celery` @@ -1106,10 +1098,7 @@ workers: # Update Strategy when Airflow Celery worker is deployed as a StatefulSet updateStrategy: ~ # Update Strategy when Airflow Celery worker is deployed as a Deployment - strategy: - rollingUpdate: - maxSurge: "100%" - maxUnavailable: "50%" + strategy: ~ # Allow relaxing ordering guarantees for Airflow Celery worker # while preserving its uniqueness and identity @@ -1124,7 +1113,7 @@ workers: # Persistence volume configuration for Airflow Celery workers persistence: # Enable persistent volumes - enabled: true + enabled: ~ # This policy determines whether PVCs should be deleted when StatefulSet is scaled down or removed persistentVolumeClaimRetentionPolicy: ~ @@ -1133,7 +1122,7 @@ workers: # whenScaled: Delete # Volume size for Airflow Celery worker StatefulSet - size: 100Gi + size: ~ # If using a custom storageClass, pass name ref to all StatefulSets here storageClassName: @@ -1141,7 +1130,7 @@ workers: # Execute init container to chown log directory. # This is currently only needed in kind, due to usage # of local-path provisioner. - fixPermissions: false + fixPermissions: ~ # Annotations to add to Airflow Celery worker volumes annotations: {} @@ -1155,7 +1144,7 @@ workers: kerberosInitContainer: # Enable kerberos init container # If workers.kerberosInitContainer.enabled is set to True, this flag has no effect - enabled: false + enabled: ~ resources: {} # limits: @@ -1187,7 +1176,7 @@ workers: kerberosInitContainer: # Enable kerberos init container # If workers.kerberosInitContainer.enabled is set to True, this flag has no effect - enabled: false + enabled: ~ resources: {} # limits: diff --git a/helm-tests/tests/helm_tests/airflow_core/test_scheduler.py b/helm-tests/tests/helm_tests/airflow_core/test_scheduler.py index 5281327db6666..20b026380567a 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_scheduler.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_scheduler.py @@ -47,68 +47,17 @@ class TestScheduler: "StatefulSet", ), ("LocalExecutor", {"celery": {"persistence": {"enabled": False}}}, "Deployment"), - # Test workers.persistence.enabled flag when celery one is default (expected no impact on kind) + # Test workers.persistence.enabled flag when celery one is default ("CeleryExecutor", {"persistence": {"enabled": False}}, "Deployment"), ("CeleryExecutor", {"persistence": {"enabled": True}}, "Deployment"), ("CeleryKubernetesExecutor", {"persistence": {"enabled": True}}, "Deployment"), ("CeleryExecutor,KubernetesExecutor", {"persistence": {"enabled": True}}, "Deployment"), ("KubernetesExecutor", {"persistence": {"enabled": True}}, "Deployment"), - ("LocalKubernetesExecutor", {"persistence": {"enabled": False}}, "StatefulSet"), + ("LocalKubernetesExecutor", {"persistence": {"enabled": False}}, "Deployment"), ("LocalKubernetesExecutor", {"persistence": {"enabled": True}}, "StatefulSet"), ("LocalExecutor", {"persistence": {"enabled": True}}, "StatefulSet"), ("LocalExecutor,KubernetesExecutor", {"persistence": {"enabled": True}}, "StatefulSet"), - ("LocalExecutor", {"persistence": {"enabled": False}}, "StatefulSet"), - # Test workers.persistence.enabled flag when celery one is unset - ( - "CeleryExecutor", - {"persistence": {"enabled": False}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "CeleryExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "CeleryKubernetesExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "CeleryExecutor,KubernetesExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "KubernetesExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "LocalKubernetesExecutor", - {"persistence": {"enabled": False}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "LocalKubernetesExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "StatefulSet", - ), - ( - "LocalExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "StatefulSet", - ), - ( - "LocalExecutor,KubernetesExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "StatefulSet", - ), - ( - "LocalExecutor", - {"persistence": {"enabled": False}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), + ("LocalExecutor", {"persistence": {"enabled": False}}, "Deployment"), ], ) def test_scheduler_kind(self, executor, workers_values, kind): diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index e41e1f0c22718..9d02691aad3c8 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -33,32 +33,11 @@ class TestWorker: ("CeleryExecutor", {"celery": {"persistence": {"enabled": True}}}, "StatefulSet"), ("CeleryKubernetesExecutor", {"celery": {"persistence": {"enabled": False}}}, "Deployment"), ("CeleryKubernetesExecutor", {"celery": {"persistence": {"enabled": True}}}, "StatefulSet"), - # Test workers.persistence.enabled flag when celery one is default (expected no impact on kind) - ("CeleryExecutor", {"persistence": {"enabled": False}}, "StatefulSet"), + # Test workers.persistence.enabled flag when celery one is default + ("CeleryExecutor", {"persistence": {"enabled": False}}, "Deployment"), ("CeleryExecutor", {"persistence": {"enabled": True}}, "StatefulSet"), - ("CeleryKubernetesExecutor", {"persistence": {"enabled": False}}, "StatefulSet"), + ("CeleryKubernetesExecutor", {"persistence": {"enabled": False}}, "Deployment"), ("CeleryKubernetesExecutor", {"persistence": {"enabled": True}}, "StatefulSet"), - # Test workers.persistence.enabled flag when celery one is unset - ( - "CeleryExecutor", - {"persistence": {"enabled": False}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "CeleryExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "StatefulSet", - ), - ( - "CeleryKubernetesExecutor", - {"persistence": {"enabled": False}, "celery": {"persistence": {"enabled": None}}}, - "Deployment", - ), - ( - "CeleryKubernetesExecutor", - {"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, - "StatefulSet", - ), ], ) def test_worker_kind(self, executor, workers_values, kind): @@ -480,16 +459,13 @@ def test_workers_update_strategy(self, workers_values, expected_update_strategy) {"celery": {"persistence": {"enabled": False}}}, {"rollingUpdate": {"maxSurge": "100%", "maxUnavailable": "50%"}}, ), - ( - {"strategy": None, "celery": {"persistence": {"enabled": False}}}, - {"rollingUpdate": {"maxSurge": "100%", "maxUnavailable": "50%"}}, - ), + ({"strategy": None, "celery": {"persistence": {"enabled": False}}}, None), ( { "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, "celery": {"persistence": {"enabled": False}}, }, - {"rollingUpdate": {"maxSurge": "100%", "maxUnavailable": "50%"}}, + {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, ), ( {"celery": {"strategy": None, "persistence": {"enabled": False}}}, @@ -497,8 +473,10 @@ def test_workers_update_strategy(self, workers_values, expected_update_strategy) ), ( { - "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, - "celery": {"strategy": None, "persistence": {"enabled": False}}, + "celery": { + "strategy": {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, + "persistence": {"enabled": False}, + } }, {"rollingUpdate": {"maxSurge": "50%", "maxUnavailable": "100%"}}, ), @@ -739,23 +717,6 @@ def test_livenessprobe_default_command(self, airflow_version, default_cmd): } }, }, - { - "livenessProbe": { - "initialDelaySeconds": 111, - "timeoutSeconds": 222, - "failureThreshold": 333, - "periodSeconds": 444, - "command": ["sh", "-c", "echo", "wow such test"], - }, - "celery": { - "livenessProbe": { - "initialDelaySeconds": None, - "timeoutSeconds": None, - "failureThreshold": None, - "periodSeconds": None, - } - }, - }, ], ) def test_livenessprobe_celery_values_overwrite(self, workers_values): @@ -793,10 +754,10 @@ def test_livenessprobe_values_overwrite(self): livenessprobe = jmespath.search("spec.template.spec.containers[0].livenessProbe", docs[0]) assert livenessprobe == { - "initialDelaySeconds": 10, - "timeoutSeconds": 20, - "failureThreshold": 5, - "periodSeconds": 60, + "initialDelaySeconds": 111, + "timeoutSeconds": 222, + "failureThreshold": 333, + "periodSeconds": 444, "exec": { "command": ["sh", "-c", "echo", "wow such test"], }, @@ -1150,17 +1111,12 @@ def test_should_not_add_command(self, workers_values): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"args": None}, ["bash", "-c", "exec \\\nairflow celery worker"]), - ({"args": []}, ["bash", "-c", "exec \\\nairflow celery worker"]), + ({"args": None}, None), + ({"args": []}, None), ({"celery": {"args": None}}, ["bash", "-c", "exec \\\nairflow celery worker"]), ({"celery": {"args": []}}, ["bash", "-c", "exec \\\nairflow celery worker"]), - ({"args": ["custom", "args"]}, ["bash", "-c", "exec \\\nairflow celery worker"]), - ( - {"args": ["custom", "{{ .Release.Service }}"]}, - ["bash", "-c", "exec \\\nairflow celery worker"], - ), - ({"celery": {"args": ["custom", "args"]}}, ["custom", "args"]), - ({"celery": {"args": ["custom", "{{ .Release.Service }}"]}}, ["custom", "Helm"]), + ({"args": ["custom", "args", "{{ .Release.Service }}"]}, ["custom", "args", "Helm"]), + ({"celery": {"args": ["custom", "args", "{{ .Release.Service }}"]}}, ["custom", "args", "Helm"]), ({"args": ["test"], "celery": {"args": ["custom", "args"]}}, ["custom", "args"]), ], ) @@ -1633,7 +1589,7 @@ def test_worker_template_storage_class_name(self, workers_values): @pytest.mark.parametrize( ("workers_values", "expected"), [ - ({"replicas": 2}, 1), + ({"replicas": 2}, 2), ({"celery": {"replicas": 2}}, 2), ({"celery": {"replicas": None}}, 1), ({"replicas": 2, "celery": {"replicas": 3}}, 3), diff --git a/helm-tests/tests/helm_tests/other/test_hpa.py b/helm-tests/tests/helm_tests/other/test_hpa.py index 5d1b5b00e7732..83bcf68b12c8a 100644 --- a/helm-tests/tests/helm_tests/other/test_hpa.py +++ b/helm-tests/tests/helm_tests/other/test_hpa.py @@ -106,10 +106,8 @@ def test_hpa_behavior(self, executor): [ ({"celery": {"persistence": {"enabled": True}}}, "StatefulSet"), ({"celery": {"persistence": {"enabled": False}}}, "Deployment"), - ({"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, "StatefulSet"), - ({"persistence": {"enabled": False}, "celery": {"persistence": {"enabled": None}}}, "Deployment"), ({"persistence": {"enabled": True}}, "StatefulSet"), - ({"persistence": {"enabled": False}}, "StatefulSet"), + ({"persistence": {"enabled": False}}, "Deployment"), ], ) def test_persistence(self, workers_persistence_values, kind): diff --git a/helm-tests/tests/helm_tests/other/test_keda.py b/helm-tests/tests/helm_tests/other/test_keda.py index 5953c7d6c23f2..4ed70c20a8b69 100644 --- a/helm-tests/tests/helm_tests/other/test_keda.py +++ b/helm-tests/tests/helm_tests/other/test_keda.py @@ -179,10 +179,8 @@ def test_keda_query_kubernetes_queue(self, executor, queue): [ ({"celery": {"persistence": {"enabled": True}}}, "StatefulSet"), ({"celery": {"persistence": {"enabled": False}}}, "Deployment"), - ({"persistence": {"enabled": True}, "celery": {"persistence": {"enabled": None}}}, "StatefulSet"), - ({"persistence": {"enabled": False}, "celery": {"persistence": {"enabled": None}}}, "Deployment"), ({"persistence": {"enabled": True}}, "StatefulSet"), - ({"persistence": {"enabled": False}}, "StatefulSet"), + ({"persistence": {"enabled": False}}, "Deployment"), ], ) def test_persistence(self, workers_persistence_values, kind):