From 7e4559aa650970f7553d3526c47ee2efec57c41b Mon Sep 17 00:00:00 2001 From: dirrao Date: Tue, 10 Sep 2024 15:48:33 +0530 Subject: [PATCH 1/5] Deprecated configuration removed --- airflow/configuration.py | 20 +------------------ .../fab/auth_manager/fab_auth_manager.py | 10 +++++++--- tests/core/test_configuration.py | 15 +++++++------- 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/airflow/configuration.py b/airflow/configuration.py index f4bea46c5c0ba..97d8258aca32f 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -326,7 +326,6 @@ def sensitive_config_values(self) -> Set[tuple[str, str]]: # noqa: UP006 # When reading new option, the old option will be checked to see if it exists. If it does a # DeprecationWarning will be issued and the old option will be used instead deprecated_options: dict[tuple[str, str], tuple[str, str, str]] = { - ("celery", "worker_precheck"): ("core", "worker_precheck", "2.0.0"), ("logging", "interleave_timestamp_parser"): ("core", "interleave_timestamp_parser", "2.6.1"), ("logging", "base_log_folder"): ("core", "base_log_folder", "2.0.0"), ("logging", "remote_logging"): ("core", "remote_logging", "2.0.0"), @@ -350,15 +349,7 @@ def sensitive_config_values(self) -> Set[tuple[str, str]]: # noqa: UP006 "2.0.0", ), ("logging", "task_log_reader"): ("core", "task_log_reader", "2.0.0"), - ("scheduler", "parsing_processes"): ("scheduler", "max_threads", "1.10.14"), - ("operators", "default_queue"): ("celery", "default_queue", "2.1.0"), - ("core", "hide_sensitive_var_conn_fields"): ("admin", "hide_sensitive_variable_fields", "2.1.0"), - ("core", "sensitive_var_conn_names"): ("admin", "sensitive_variable_fields", "2.1.0"), - ("core", "default_pool_task_slot_count"): ("core", "non_pooled_task_slot_count", "1.10.4"), - ("core", "max_active_tasks_per_dag"): ("core", "dag_concurrency", "2.2.0"), ("logging", "worker_log_server_port"): ("celery", "worker_log_server_port", "2.2.0"), - ("api", "access_control_allow_origins"): ("api", "access_control_allow_origin", "2.2.0"), - ("api", "auth_backends"): ("api", "auth_backend", "2.3.0"), ("database", "sql_alchemy_conn"): ("core", "sql_alchemy_conn", "2.3.0"), ("database", "sql_engine_encoding"): ("core", "sql_engine_encoding", "2.3.0"), ("database", "sql_engine_collation_for_ids"): ("core", "sql_engine_collation_for_ids", "2.3.0"), @@ -371,19 +362,10 @@ def sensitive_config_values(self) -> Set[tuple[str, str]]: # noqa: UP006 ("database", "sql_alchemy_connect_args"): ("core", "sql_alchemy_connect_args", "2.3.0"), ("database", "load_default_connections"): ("core", "load_default_connections", "2.3.0"), ("database", "max_db_retries"): ("core", "max_db_retries", "2.3.0"), - ("scheduler", "parsing_cleanup_interval"): ("scheduler", "deactivate_stale_dags_interval", "2.5.0"), - ("scheduler", "task_queued_timeout_check_interval"): ( - "kubernetes_executor", - "worker_pods_pending_timeout_check_interval", - "2.6.0", - ), - ("fab", "update_fab_perms"): ("webserver", "update_fab_perms", "2.9.0"), - ("fab", "auth_rate_limited"): ("webserver", "auth_rate_limited", "2.9.0"), - ("fab", "auth_rate_limit"): ("webserver", "auth_rate_limit", "2.9.0"), } # A mapping of new section -> (old section, since_version). - deprecated_sections: dict[str, tuple[str, str]] = {"kubernetes_executor": ("kubernetes", "2.5.0")} + deprecated_sections: dict[str, tuple[str, str]] = {} # Now build the inverse so we can go from old_section/old_key to new_section/new_key # if someone tries to retrieve it based on old_section/old_key diff --git a/airflow/providers/fab/auth_manager/fab_auth_manager.py b/airflow/providers/fab/auth_manager/fab_auth_manager.py index 85e0687b93b43..4e91d6d2d1470 100644 --- a/airflow/providers/fab/auth_manager/fab_auth_manager.py +++ b/airflow/providers/fab/auth_manager/fab_auth_manager.py @@ -522,9 +522,13 @@ def _sync_appbuilder_roles(self): # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. - if conf.getboolean( - "fab", "UPDATE_FAB_PERMS", fallback=conf.getboolean("webserver", "UPDATE_FAB_PERMS") - ): + from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + + if AIRFLOW_V_3_0_PLUS: + fallback = None + else: + fallback = conf.getboolean("webserver", "UPDATE_FAB_PERMS") + if conf.getboolean("fab", "UPDATE_FAB_PERMS", fallback=fallback): self.security_manager.sync_roles() diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index d02e9b4b3cb82..d11f325a33b4b 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -53,6 +53,13 @@ HOME_DIR = os.path.expanduser("~") +# The conf has been updated with deactivate_stale_dags_interval to test the functionality of deprecated options support. +conf.deprecated_options[("scheduler", "parsing_cleanup_interval")] = ( + "scheduler", + "deactivate_stale_dags_interval", + "2.5.0", +) + @pytest.fixture(scope="module", autouse=True) def restore_env(): @@ -1029,14 +1036,6 @@ def test_deprecated_values_from_conf(self): @pytest.mark.parametrize( "old, new", [ - ( - ("api", "auth_backend", "airflow.providers.fab.auth_manager.api.auth.backend.basic_auth"), - ( - "api", - "auth_backends", - "airflow.providers.fab.auth_manager.api.auth.backend.basic_auth,airflow.api.auth.backend.session", - ), - ), ( ("core", "sql_alchemy_conn", "postgres+psycopg2://localhost/postgres"), ("database", "sql_alchemy_conn", "postgresql://localhost/postgres"), From a416a67a66e5a232016ceafabc762018b160a524 Mon Sep 17 00:00:00 2001 From: dirrao Date: Tue, 10 Sep 2024 16:01:46 +0530 Subject: [PATCH 2/5] news fragment added --- newsfragments/42129.significant.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 newsfragments/42129.significant.rst diff --git a/newsfragments/42129.significant.rst b/newsfragments/42129.significant.rst new file mode 100644 index 0000000000000..f7cd28e9bdb16 --- /dev/null +++ b/newsfragments/42129.significant.rst @@ -0,0 +1,17 @@ +Removed deprecated database configuration. + + * Removed deprecated configuration ``worker_precheck`` from ``core``. Please use ``worker_precheck`` from ``celery`` instead. + * Removed deprecated configuration ``max_threads`` from ``scheduler``. Please use ``parsing_processes`` from ``scheduler`` instead. + * Removed deprecated configuration ``default_queue`` from ``celery``. Please use ``default_queue`` from ``operators`` instead. + * Removed deprecated configuration ``hide_sensitive_variable_fields`` from ``admin``. Please use ``hide_sensitive_var_conn_fields`` from ``core`` instead. + * Removed deprecated configuration ``sensitive_variable_fields`` from ``admin``. Please use ``sensitive_var_conn_names`` from ``core`` instead. + * Removed deprecated configuration ``non_pooled_task_slot_count`` from ``core``. Please use ``default_pool_task_slot_count`` from ``core`` instead. + * Removed deprecated configuration ``dag_concurrency`` from ``core``. Please use ``max_active_tasks_per_dag`` from ``core`` instead. + * Removed deprecated configuration ``access_control_allow_origin`` from ``api``. Please use ``access_control_allow_origins`` from ``api`` instead. + * Removed deprecated configuration ``auth_backend`` from ``api``. Please use ``auth_backends`` from ``api`` instead. + * Removed deprecated configuration ``deactivate_stale_dags_interval`` from ``scheduler``. Please use ``parsing_cleanup_interval`` from ``scheduler`` instead. + * Removed deprecated configuration ``worker_pods_pending_timeout_check_interval`` from ``kubernetes_executor``. Please use ``task_queued_timeout_check_interval`` from ``scheduler`` instead. + * Removed deprecated configuration ``update_fab_perms`` from ``webserver``. Please use ``update_fab_perms`` from ``fab`` instead. + * Removed deprecated configuration ``auth_rate_limited`` from ``webserver``. Please use ``auth_rate_limited`` from ``fab`` instead. + * Removed deprecated configuration ``auth_rate_limit`` from ``webserver``. Please use ``auth_rate_limit`` from ``fab`` instead. + * Removed deprecated configuration section ``kubernetes``. Please use ``kubernetes_executor`` instead. From 23a641b0697c29a7a9806f324d3abc50135e22fd Mon Sep 17 00:00:00 2001 From: dirrao Date: Tue, 10 Sep 2024 16:03:49 +0530 Subject: [PATCH 3/5] news fragment added --- newsfragments/42129.significant.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/42129.significant.rst b/newsfragments/42129.significant.rst index f7cd28e9bdb16..06a5da7fc889a 100644 --- a/newsfragments/42129.significant.rst +++ b/newsfragments/42129.significant.rst @@ -1,4 +1,4 @@ -Removed deprecated database configuration. +Removed deprecated configuration. * Removed deprecated configuration ``worker_precheck`` from ``core``. Please use ``worker_precheck`` from ``celery`` instead. * Removed deprecated configuration ``max_threads`` from ``scheduler``. Please use ``parsing_processes`` from ``scheduler`` instead. From ff0daa3d083b73da1f995e1433d581b5848d3e07 Mon Sep 17 00:00:00 2001 From: dirrao Date: Tue, 10 Sep 2024 22:52:26 +0530 Subject: [PATCH 4/5] Deprecated configuration removed --- .../cncf/kubernetes/executors/test_kubernetes_executor.py | 6 +++--- tests/providers/cncf/kubernetes/test_client.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py index 769eb9c980ae0..ca54738651539 100644 --- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -1227,8 +1227,8 @@ def test_kube_config_get_namespace_list( self, raw_multi_namespace_mode, raw_value_namespace_list, expected_value_in_kube_config ): config = { - ("kubernetes", "multi_namespace_mode"): raw_multi_namespace_mode, - ("kubernetes", "multi_namespace_mode_namespace_list"): raw_value_namespace_list, + ("kubernetes_executor", "multi_namespace_mode"): raw_multi_namespace_mode, + ("kubernetes_executor", "multi_namespace_mode_namespace_list"): raw_value_namespace_list, } with conf_vars(config): executor = KubernetesExecutor() @@ -1504,7 +1504,7 @@ def test_annotations_for_logging_task_metadata(self): } get_logs_task_metadata.cache_clear() try: - with conf_vars({("kubernetes", "logs_task_metadata"): "True"}): + with conf_vars({("kubernetes_executor", "logs_task_metadata"): "True"}): expected_annotations = { "dag_id": "dag", "run_id": "run_id", diff --git a/tests/providers/cncf/kubernetes/test_client.py b/tests/providers/cncf/kubernetes/test_client.py index 836bdefdac407..1384068fd2860 100644 --- a/tests/providers/cncf/kubernetes/test_client.py +++ b/tests/providers/cncf/kubernetes/test_client.py @@ -92,7 +92,9 @@ def test_disable_verify_ssl(self): assert not configuration.verify_ssl @mock.patch("kubernetes.config.incluster_config.InClusterConfigLoader") - @conf_vars({("kubernetes", "api_client_retry_configuration"): '{"total": 3, "backoff_factor": 0.5}'}) + @conf_vars( + {("kubernetes_executor", "api_client_retry_configuration"): '{"total": 3, "backoff_factor": 0.5}'} + ) def test_api_client_retry_configuration_correct_values(self, mock_in_cluster_loader): get_kube_client(in_cluster=True) client_configuration = mock_in_cluster_loader().load_and_set.call_args.args[0] From c497885001073fb23e78ea5bd5e6143a6f23c2d5 Mon Sep 17 00:00:00 2001 From: dirrao Date: Sat, 14 Sep 2024 11:02:56 +0530 Subject: [PATCH 5/5] Deprecated configuration removed --- .../cncf/kubernetes/executors/kubernetes_executor.py | 4 +++- airflow/providers/fab/auth_manager/fab_auth_manager.py | 6 ++++-- .../cncf/kubernetes/executors/test_kubernetes_executor.py | 8 ++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py index 754d835dc3481..0b2de8085c5dd 100644 --- a/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +++ b/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py @@ -140,7 +140,9 @@ def __init__(self): self.last_handled: dict[TaskInstanceKey, float] = {} self.kubernetes_queue: str | None = None self.task_publish_retries: Counter[TaskInstanceKey] = Counter() - self.task_publish_max_retries = conf.getint("kubernetes", "task_publish_max_retries", fallback=0) + self.task_publish_max_retries = conf.getint( + "kubernetes_executor", "task_publish_max_retries", fallback=0 + ) super().__init__(parallelism=self.kube_config.parallelism) def _list_pods(self, query_kwargs): diff --git a/airflow/providers/fab/auth_manager/fab_auth_manager.py b/airflow/providers/fab/auth_manager/fab_auth_manager.py index 4e91d6d2d1470..56055589104e1 100644 --- a/airflow/providers/fab/auth_manager/fab_auth_manager.py +++ b/airflow/providers/fab/auth_manager/fab_auth_manager.py @@ -522,9 +522,11 @@ def _sync_appbuilder_roles(self): # Otherwise, when the name of a view or menu is changed, the framework # will add the new Views and Menus names to the backend, but will not # delete the old ones. - from tests.test_utils.compat import AIRFLOW_V_3_0_PLUS + from packaging.version import Version - if AIRFLOW_V_3_0_PLUS: + from airflow.version import version + + if Version(Version(version).base_version) >= Version("3.0.0"): fallback = None else: fallback = conf.getboolean("webserver", "UPDATE_FAB_PERMS") diff --git a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py index ca54738651539..8cc46c3dba76b 100644 --- a/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py +++ b/tests/providers/cncf/kubernetes/executors/test_kubernetes_executor.py @@ -413,7 +413,7 @@ def test_run_next_exception_requeue( mock_api_client.sanitize_for_serialization.return_value = {} mock_kube_client.api_client = mock_api_client config = { - ("kubernetes", "pod_template_file"): template_file, + ("kubernetes_executor", "pod_template_file"): template_file, } with conf_vars(config): kubernetes_executor = self.kubernetes_executor @@ -513,7 +513,7 @@ def test_run_next_pod_reconciliation_error( mock_api_client = mock.MagicMock() mock_api_client.sanitize_for_serialization.return_value = {} mock_kube_client.api_client = mock_api_client - config = {("kubernetes", "pod_template_file"): template_file} + config = {("kubernetes_executor", "pod_template_file"): template_file} with conf_vars(config): kubernetes_executor = self.kubernetes_executor kubernetes_executor.start() @@ -597,7 +597,7 @@ def test_pod_template_file_override_in_executor_config( mock_kube_client = mock.patch("kubernetes.client.CoreV1Api", autospec=True) mock_get_kube_client.return_value = mock_kube_client - with conf_vars({("kubernetes", "pod_template_file"): None}): + with conf_vars({("kubernetes_executor", "pod_template_file"): None}): executor = self.kubernetes_executor executor.start() try: @@ -1525,7 +1525,7 @@ def test_annotations_for_logging_task_metadata_fallback(self): } get_logs_task_metadata.cache_clear() try: - with conf_vars({("kubernetes", "logs_task_metadata"): "False"}): + with conf_vars({("kubernetes_executor", "logs_task_metadata"): "False"}): expected_annotations = "" annotations_actual = annotations_for_logging_task_metadata(annotations_test) assert annotations_actual == expected_annotations