From f6213f9cd6f7e94913f3524822e71ba0e9db3aaa Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 23 Oct 2024 21:16:47 +0530 Subject: [PATCH 1/2] Masking configuration values irrelevant to DAG author (#43040) Some configurations are irrelevant to DAG authors and hence we need to mask those to avoid it from getting logged unknowingly. Co-authored-by: adesai Co-authored-by: Ash Berlin-Taylor (cherry picked from commit 0b030c562363dd924bbbee0793636be18deeabe3) --- airflow/configuration.py | 15 +++++++++++++++ airflow/settings.py | 3 +++ tests/core/test_configuration.py | 15 +++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/airflow/configuration.py b/airflow/configuration.py index 618f5185db7d6..22e2c6abf702d 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -851,6 +851,21 @@ def _create_future_warning(name: str, section: str, current_value: Any, new_valu stacklevel=3, ) + def mask_secrets(self): + from airflow.utils.log.secrets_masker import mask_secret + + for section, key in self.sensitive_config_values: + try: + value = self.get(section, key) + except AirflowConfigException: + log.debug( + "Could not retrieve value from section %s, for key %s. Skipping redaction of this conf.", + section, + key, + ) + continue + mask_secret(value) + def _env_var_name(self, section: str, key: str) -> str: return f"{ENV_VAR_PREFIX}{section.replace('.', '_').upper()}__{key.upper()}" diff --git a/airflow/settings.py b/airflow/settings.py index dc24a2c5acc5a..7e9626d788f50 100644 --- a/airflow/settings.py +++ b/airflow/settings.py @@ -790,6 +790,9 @@ def initialize(): configure_orm() configure_action_logging() + # mask the sensitive_config_values + conf.mask_secrets() + # Run any custom runtime checks that needs to be executed for providers run_providers_custom_runtime_checks() diff --git a/tests/core/test_configuration.py b/tests/core/test_configuration.py index 62548a3f26688..b200d16baad8a 100644 --- a/tests/core/test_configuration.py +++ b/tests/core/test_configuration.py @@ -1785,3 +1785,18 @@ def test_config_paths_is_directory(self): with pytest.raises(IsADirectoryError, match="configuration file, but got a directory"): write_default_airflow_configuration_if_needed() + + @conf_vars({("mysection1", "mykey1"): "supersecret1", ("mysection2", "mykey2"): "supersecret2"}) + @patch.object( + conf, + "sensitive_config_values", + new_callable=lambda: [("mysection1", "mykey1"), ("mysection2", "mykey2")], + ) + @patch("airflow.utils.log.secrets_masker.mask_secret") + def test_mask_conf_values(self, mock_mask_secret, mock_sensitive_config_values): + conf.mask_secrets() + + mock_mask_secret.assert_any_call("supersecret1") + mock_mask_secret.assert_any_call("supersecret2") + + assert mock_mask_secret.call_count == 2 From 1ebb26a65fcbee65effd84c55b8fc40d55d73102 Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 24 Oct 2024 01:01:42 +0100 Subject: [PATCH 2/2] Suppress warnings when masking sensitive confs (#43335) This is to prevent issues such as https://github.com/apache/airflow/pull/43334 (cherry picked from commit 0a9c3c007bf1672398a63bb9c97b82ec48f60afc) --- airflow/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/configuration.py b/airflow/configuration.py index 22e2c6abf702d..81eb0fc725344 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -856,7 +856,7 @@ def mask_secrets(self): for section, key in self.sensitive_config_values: try: - value = self.get(section, key) + value = self.get(section, key, suppress_warnings=True) except AirflowConfigException: log.debug( "Could not retrieve value from section %s, for key %s. Skipping redaction of this conf.",