From a7aff334994edeba08d3112f0047b6507396d539 Mon Sep 17 00:00:00 2001 From: bugraoz93 Date: Sun, 9 Nov 2025 19:30:52 +0100 Subject: [PATCH 1/5] Add dummy Fernet key for integration tests --- airflow-core/docs/howto/docker-compose/docker-compose.yaml | 6 +++++- task-sdk-integration-tests/docker-compose.yaml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index 3892e04414358..273048d7380d6 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -60,7 +60,11 @@ x-airflow-common: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 - AIRFLOW__CORE__FERNET_KEY: '' + # + # Please Do not use this Fernet key in any deployments! Please generate your own key. + # This is specifically generated for tests and not as default. It could fail any static level code checks. + # + AIRFLOW__CORE__FERNET_KEY: 'dGVzdF9rZXlfdGVzdF9rZXlfdGVzdF9rZXlfdGVzdF8=' AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' AIRFLOW__CORE__LOAD_EXAMPLES: 'true' AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' diff --git a/task-sdk-integration-tests/docker-compose.yaml b/task-sdk-integration-tests/docker-compose.yaml index f2f2af37cdedc..3f81bee5df4d7 100644 --- a/task-sdk-integration-tests/docker-compose.yaml +++ b/task-sdk-integration-tests/docker-compose.yaml @@ -26,7 +26,11 @@ x-airflow-common: AIRFLOW__CORE__AUTH_MANAGER: 'airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager' AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: 'true' AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow - AIRFLOW__CORE__FERNET_KEY: '' + # + # Please Do not use this Fernet key in any deployments! Please generate your own key. + # This is specifically generated for tests and not as default. It could fail any static level code checks. + # + AIRFLOW__CORE__FERNET_KEY: 'dGVzdF9rZXlfdGVzdF9rZXlfdGVzdF9rZXlfdGVzdF8=' AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__CORE__DAGS_FOLDER: '/opt/airflow/dags' AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' From fdf8a02ae441c1c0a606ef12f156521ad3e1057b Mon Sep 17 00:00:00 2001 From: bugraoz93 Date: Tue, 11 Nov 2025 23:35:46 +0100 Subject: [PATCH 2/5] dynamic fernet key generation from devel-common --- .../howto/docker-compose/docker-compose.yaml | 6 +-- .../tests/airflowctl_tests/conftest.py | 4 ++ .../tests/airflow_e2e_tests/conftest.py | 4 ++ .../src/tests_common/test_utils/fernet.py | 44 +++++++++++++++ .../tests_common/test_utils/test_fernet.py | 54 +++++++++++++++++++ .../docker-compose.yaml | 6 +-- .../tests/task_sdk_tests/conftest.py | 2 + 7 files changed, 110 insertions(+), 10 deletions(-) create mode 100644 devel-common/src/tests_common/test_utils/fernet.py create mode 100644 devel-common/tests/unit/tests_common/test_utils/test_fernet.py diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index 273048d7380d6..3892e04414358 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -60,11 +60,7 @@ x-airflow-common: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 - # - # Please Do not use this Fernet key in any deployments! Please generate your own key. - # This is specifically generated for tests and not as default. It could fail any static level code checks. - # - AIRFLOW__CORE__FERNET_KEY: 'dGVzdF9rZXlfdGVzdF9rZXlfdGVzdF9rZXlfdGVzdF8=' + AIRFLOW__CORE__FERNET_KEY: '' AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' AIRFLOW__CORE__LOAD_EXAMPLES: 'true' AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' diff --git a/airflow-ctl-tests/tests/airflowctl_tests/conftest.py b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py index 13dacf5d8c50e..335c8ff2ce309 100644 --- a/airflow-ctl-tests/tests/airflowctl_tests/conftest.py +++ b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py @@ -30,6 +30,8 @@ DOCKER_IMAGE, ) +from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml + docker_client = None @@ -166,6 +168,8 @@ def docker_compose_up(tmp_path_factory): os.environ["AIRFLOW_CTL_VERSION"] = os.environ.get("AIRFLOW_CTL_VERSION", "1.0.0") os.environ["ENV_FILE_PATH"] = str(tmp_dir / ".env") + update_environment_variable_from_compose_yaml(file_path=tmp_docker_compose_file) + # Initialize Docker client docker_client = DockerClient(compose_files=[str(tmp_docker_compose_file)]) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 20a0dd2fcbbf8..3c58d840053ac 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -37,6 +37,8 @@ TEST_REPORT_FILE, ) +from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml + console = Console(width=400, color_system="standard") compose_instance = None airflow_logs_path = None @@ -99,6 +101,8 @@ def spin_up_airflow_environment(tmp_path_factory): pull = False if DOCKER_IMAGE.startswith("ghcr.io/apache/airflow/main/") else True try: + update_environment_variable_from_compose_yaml(file_path=compose_file_names.pop()) + console.print(f"[blue]Spinning up airflow environment using {DOCKER_IMAGE}") compose_instance = DockerCompose(tmp_dir, compose_file_name=compose_file_names, pull=pull) diff --git a/devel-common/src/tests_common/test_utils/fernet.py b/devel-common/src/tests_common/test_utils/fernet.py new file mode 100644 index 0000000000000..0b68674b78879 --- /dev/null +++ b/devel-common/src/tests_common/test_utils/fernet.py @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pathlib import Path + +from cryptography.fernet import Fernet + + +def generate_fernet_key_string() -> str: + """Generate a new Fernet key.""" + return Fernet.generate_key().decode() + + +def update_environment_variable_from_compose_yaml(file_path: str | Path) -> None: + """Update environment variable AIRFLOW__CORE__FERNET_KEY for a given docker-compose YAML file.""" + from ruamel.yaml import YAML + + yaml = YAML() + file_path = Path(file_path) + with file_path.open("r") as f: + compose_yaml = yaml.load(f) + + x_airflow_common = compose_yaml.get("x-airflow-common", {}) + environment = x_airflow_common.get("environment", {}) + environment["AIRFLOW__CORE__FERNET_KEY"] = generate_fernet_key_string() + + with file_path.open("w") as f: + yaml.dump(compose_yaml, f) diff --git a/devel-common/tests/unit/tests_common/test_utils/test_fernet.py b/devel-common/tests/unit/tests_common/test_utils/test_fernet.py new file mode 100644 index 0000000000000..728f809a1f4d5 --- /dev/null +++ b/devel-common/tests/unit/tests_common/test_utils/test_fernet.py @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest.mock import patch + +from tests_common.test_utils.fernet import generate_fernet_key_string + + +class TestFernetUtils: + """Test utils for Fernet encryption.""" + + @patch("tests_common.test_utils.fernet.Fernet") + def test_generate_fernet_key_string(self, mock_fernet): + """Test generating a Fernet key.""" + mock_fernet.generate_key.return_value = b"test_key" + key = generate_fernet_key_string() + assert key == "test_key" + mock_fernet.generate_key.assert_called_once() + + @patch("tests_common.test_utils.fernet.Fernet") + def test_update_environment_variable_from_compose_yaml(self, mock_fernet, tmp_path): + """Test updating environment variable in docker-compose YAML file.""" + mock_fernet.generate_key.return_value = b"new_test_key" + compose_yaml_content = """ + x-airflow-common: + environment: + AIRFLOW__CORE__FERNET_KEY: old_key + """ + compose_file = tmp_path / "docker-compose.yaml" + compose_file.write_text(compose_yaml_content) + + from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml + + update_environment_variable_from_compose_yaml(compose_file) + + updated_content = compose_file.read_text() + assert "new_test_key" in updated_content + mock_fernet.generate_key.assert_called_once() diff --git a/task-sdk-integration-tests/docker-compose.yaml b/task-sdk-integration-tests/docker-compose.yaml index 3f81bee5df4d7..f2f2af37cdedc 100644 --- a/task-sdk-integration-tests/docker-compose.yaml +++ b/task-sdk-integration-tests/docker-compose.yaml @@ -26,11 +26,7 @@ x-airflow-common: AIRFLOW__CORE__AUTH_MANAGER: 'airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager' AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: 'true' AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow - # - # Please Do not use this Fernet key in any deployments! Please generate your own key. - # This is specifically generated for tests and not as default. It could fail any static level code checks. - # - AIRFLOW__CORE__FERNET_KEY: 'dGVzdF9rZXlfdGVzdF9rZXlfdGVzdF9rZXlfdGVzdF8=' + AIRFLOW__CORE__FERNET_KEY: '' AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__CORE__DAGS_FOLDER: '/opt/airflow/dags' AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' diff --git a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py index 19d410572e9cf..fd530f1409f23 100644 --- a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py +++ b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py @@ -43,6 +43,8 @@ TASK_SDK_INTEGRATION_LOCAL_DOCKER_COMPOSE_FILE_PATH, ) +from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml + def print_diagnostics(compose, compose_version, docker_version): """Print diagnostic information when test fails.""" From af90838efe4bd568ac748c1400b422b0944373d1 Mon Sep 17 00:00:00 2001 From: bugraoz93 Date: Wed, 12 Nov 2025 01:12:04 +0100 Subject: [PATCH 3/5] Add pyyaml dependency and update YAML loading method in fernet.py --- devel-common/pyproject.toml | 1 + .../src/tests_common/test_utils/fernet.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml index ad1c8cf371252..b8c4de99247fe 100644 --- a/devel-common/pyproject.toml +++ b/devel-common/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "typer-slim>=0.15.1", "time-machine>=2.15.0", "wheel>=0.42.0", + "pyyaml>=6.0.3", "yamllint>=1.33.0", "python-on-whales>=0.70.0", "apache-airflow-devel-common[basic]" diff --git a/devel-common/src/tests_common/test_utils/fernet.py b/devel-common/src/tests_common/test_utils/fernet.py index 0b68674b78879..c53fff8001fac 100644 --- a/devel-common/src/tests_common/test_utils/fernet.py +++ b/devel-common/src/tests_common/test_utils/fernet.py @@ -19,6 +19,7 @@ from pathlib import Path +import yaml from cryptography.fernet import Fernet @@ -29,16 +30,19 @@ def generate_fernet_key_string() -> str: def update_environment_variable_from_compose_yaml(file_path: str | Path) -> None: """Update environment variable AIRFLOW__CORE__FERNET_KEY for a given docker-compose YAML file.""" - from ruamel.yaml import YAML - - yaml = YAML() - file_path = Path(file_path) + file_path = Path(file_path) if isinstance(file_path, str) else file_path with file_path.open("r") as f: - compose_yaml = yaml.load(f) + compose_yaml = yaml.safe_load(f) x_airflow_common = compose_yaml.get("x-airflow-common", {}) + if x_airflow_common == {}: + raise ValueError( + "x-airflow-common was not found in a docker-compose file, please either add it or update here." + ) environment = x_airflow_common.get("environment", {}) environment["AIRFLOW__CORE__FERNET_KEY"] = generate_fernet_key_string() + x_airflow_common["environment"] = environment + compose_yaml["x-airflow-common"] = x_airflow_common with file_path.open("w") as f: - yaml.dump(compose_yaml, f) + yaml.safe_dump(compose_yaml, f) From 83f0031449b1cb66c2e0b59bd936d3997332de19 Mon Sep 17 00:00:00 2001 From: bugraoz93 Date: Sat, 15 Nov 2025 21:24:54 +0100 Subject: [PATCH 4/5] Revert integration test changes --- airflow-ctl-tests/tests/airflowctl_tests/conftest.py | 4 ---- airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py | 4 ---- task-sdk-integration-tests/tests/task_sdk_tests/conftest.py | 2 -- 3 files changed, 10 deletions(-) diff --git a/airflow-ctl-tests/tests/airflowctl_tests/conftest.py b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py index 335c8ff2ce309..13dacf5d8c50e 100644 --- a/airflow-ctl-tests/tests/airflowctl_tests/conftest.py +++ b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py @@ -30,8 +30,6 @@ DOCKER_IMAGE, ) -from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml - docker_client = None @@ -168,8 +166,6 @@ def docker_compose_up(tmp_path_factory): os.environ["AIRFLOW_CTL_VERSION"] = os.environ.get("AIRFLOW_CTL_VERSION", "1.0.0") os.environ["ENV_FILE_PATH"] = str(tmp_dir / ".env") - update_environment_variable_from_compose_yaml(file_path=tmp_docker_compose_file) - # Initialize Docker client docker_client = DockerClient(compose_files=[str(tmp_docker_compose_file)]) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 3c58d840053ac..20a0dd2fcbbf8 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -37,8 +37,6 @@ TEST_REPORT_FILE, ) -from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml - console = Console(width=400, color_system="standard") compose_instance = None airflow_logs_path = None @@ -101,8 +99,6 @@ def spin_up_airflow_environment(tmp_path_factory): pull = False if DOCKER_IMAGE.startswith("ghcr.io/apache/airflow/main/") else True try: - update_environment_variable_from_compose_yaml(file_path=compose_file_names.pop()) - console.print(f"[blue]Spinning up airflow environment using {DOCKER_IMAGE}") compose_instance = DockerCompose(tmp_dir, compose_file_name=compose_file_names, pull=pull) diff --git a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py index fd530f1409f23..19d410572e9cf 100644 --- a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py +++ b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py @@ -43,8 +43,6 @@ TASK_SDK_INTEGRATION_LOCAL_DOCKER_COMPOSE_FILE_PATH, ) -from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml - def print_diagnostics(compose, compose_version, docker_version): """Print diagnostic information when test fails.""" From faf79d24aec1f45238841f855c1894c3425cac5a Mon Sep 17 00:00:00 2001 From: bugraoz93 Date: Sun, 16 Nov 2025 00:34:48 +0100 Subject: [PATCH 5/5] Update integration tests to use always the same generated Fernet key from environment variable --- .../howto/docker-compose/docker-compose.yaml | 2 +- .../tests/airflowctl_tests/conftest.py | 9 ++++- .../tests/airflow_e2e_tests/conftest.py | 8 +++++ devel-common/pyproject.toml | 1 - .../src/tests_common/test_utils/fernet.py | 33 ++++--------------- .../tests_common/test_utils/test_fernet.py | 33 +++---------------- .../docker-compose.yaml | 4 ++- .../tests/task_sdk_tests/conftest.py | 8 +++++ 8 files changed, 39 insertions(+), 59 deletions(-) diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index 3892e04414358..75dac6fb7543f 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -60,7 +60,7 @@ x-airflow-common: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 - AIRFLOW__CORE__FERNET_KEY: '' + AIRFLOW__CORE__FERNET_KEY: ${FERNET_KEY} AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' AIRFLOW__CORE__LOAD_EXAMPLES: 'true' AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' diff --git a/airflow-ctl-tests/tests/airflowctl_tests/conftest.py b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py index 13dacf5d8c50e..163b4731551bf 100644 --- a/airflow-ctl-tests/tests/airflowctl_tests/conftest.py +++ b/airflow-ctl-tests/tests/airflowctl_tests/conftest.py @@ -30,13 +30,15 @@ DOCKER_IMAGE, ) +from tests_common.test_utils.fernet import generate_fernet_key_string + docker_client = None # Pytest hook to run at the start of the session def pytest_sessionstart(session): """Install airflowctl at the very start of the pytest session.""" - airflow_ctl_version = os.environ.get("AIRFLOW_CTL_VERSION", "1.0.0") + airflow_ctl_version = os.environ.get("AIRFLOW_CTL_VERSION", "0.1.0") console.print(f"[yellow]Installing apache-airflow-ctl=={airflow_ctl_version} via pytest_sessionstart...") airflow_ctl_path = AIRFLOW_ROOT_PATH / "airflow-ctl" @@ -165,6 +167,11 @@ def docker_compose_up(tmp_path_factory): os.environ["AIRFLOW_IMAGE_NAME"] = DOCKER_IMAGE os.environ["AIRFLOW_CTL_VERSION"] = os.environ.get("AIRFLOW_CTL_VERSION", "1.0.0") os.environ["ENV_FILE_PATH"] = str(tmp_dir / ".env") + # + # Please Do not use this Fernet key in any deployments! Please generate your own key. + # This is specifically generated for integration tests and not as default. + # + os.environ["FERNET_KEY"] = generate_fernet_key_string() # Initialize Docker client docker_client = DockerClient(compose_files=[str(tmp_docker_compose_file)]) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index 20a0dd2fcbbf8..2886ca58b863c 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -37,6 +37,8 @@ TEST_REPORT_FILE, ) +from tests_common.test_utils.fernet import generate_fernet_key_string + console = Console(width=400, color_system="standard") compose_instance = None airflow_logs_path = None @@ -94,6 +96,12 @@ def spin_up_airflow_environment(tmp_path_factory): compose_file_names.append("localstack.yml") _setup_s3_integration(dot_env_file, tmp_dir) + # + # Please Do not use this Fernet key in any deployments! Please generate your own key. + # This is specifically generated for integration tests and not as default. + # + os.environ["FERNET_KEY"] = generate_fernet_key_string() + # If we are using the image from ghcr.io/apache/airflow/main we do not pull # as it is already available and loaded using prepare_breeze_and_image step in workflow pull = False if DOCKER_IMAGE.startswith("ghcr.io/apache/airflow/main/") else True diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml index b8c4de99247fe..ad1c8cf371252 100644 --- a/devel-common/pyproject.toml +++ b/devel-common/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ "typer-slim>=0.15.1", "time-machine>=2.15.0", "wheel>=0.42.0", - "pyyaml>=6.0.3", "yamllint>=1.33.0", "python-on-whales>=0.70.0", "apache-airflow-devel-common[basic]" diff --git a/devel-common/src/tests_common/test_utils/fernet.py b/devel-common/src/tests_common/test_utils/fernet.py index c53fff8001fac..e082763a94ae4 100644 --- a/devel-common/src/tests_common/test_utils/fernet.py +++ b/devel-common/src/tests_common/test_utils/fernet.py @@ -17,32 +17,11 @@ # under the License. from __future__ import annotations -from pathlib import Path +import base64 +import hashlib -import yaml -from cryptography.fernet import Fernet - -def generate_fernet_key_string() -> str: - """Generate a new Fernet key.""" - return Fernet.generate_key().decode() - - -def update_environment_variable_from_compose_yaml(file_path: str | Path) -> None: - """Update environment variable AIRFLOW__CORE__FERNET_KEY for a given docker-compose YAML file.""" - file_path = Path(file_path) if isinstance(file_path, str) else file_path - with file_path.open("r") as f: - compose_yaml = yaml.safe_load(f) - - x_airflow_common = compose_yaml.get("x-airflow-common", {}) - if x_airflow_common == {}: - raise ValueError( - "x-airflow-common was not found in a docker-compose file, please either add it or update here." - ) - environment = x_airflow_common.get("environment", {}) - environment["AIRFLOW__CORE__FERNET_KEY"] = generate_fernet_key_string() - x_airflow_common["environment"] = environment - compose_yaml["x-airflow-common"] = x_airflow_common - - with file_path.open("w") as f: - yaml.safe_dump(compose_yaml, f) +def generate_fernet_key_string(string_key: str = "AIRFLOW_INTEGRATION_TEST") -> str: + """Generate always the same Fernet key value as a URL-safe base64-encoded 32-byte key.""" + raw = hashlib.sha256(string_key.encode()).digest() # 32 bytes + return base64.urlsafe_b64encode(raw).decode() diff --git a/devel-common/tests/unit/tests_common/test_utils/test_fernet.py b/devel-common/tests/unit/tests_common/test_utils/test_fernet.py index 728f809a1f4d5..b459da3126e92 100644 --- a/devel-common/tests/unit/tests_common/test_utils/test_fernet.py +++ b/devel-common/tests/unit/tests_common/test_utils/test_fernet.py @@ -17,38 +17,15 @@ # under the License. from __future__ import annotations -from unittest.mock import patch - from tests_common.test_utils.fernet import generate_fernet_key_string class TestFernetUtils: """Test utils for Fernet encryption.""" - @patch("tests_common.test_utils.fernet.Fernet") - def test_generate_fernet_key_string(self, mock_fernet): + def test_generate_fernet_key_string(self): """Test generating a Fernet key.""" - mock_fernet.generate_key.return_value = b"test_key" - key = generate_fernet_key_string() - assert key == "test_key" - mock_fernet.generate_key.assert_called_once() - - @patch("tests_common.test_utils.fernet.Fernet") - def test_update_environment_variable_from_compose_yaml(self, mock_fernet, tmp_path): - """Test updating environment variable in docker-compose YAML file.""" - mock_fernet.generate_key.return_value = b"new_test_key" - compose_yaml_content = """ - x-airflow-common: - environment: - AIRFLOW__CORE__FERNET_KEY: old_key - """ - compose_file = tmp_path / "docker-compose.yaml" - compose_file.write_text(compose_yaml_content) - - from tests_common.test_utils.fernet import update_environment_variable_from_compose_yaml - - update_environment_variable_from_compose_yaml(compose_file) - - updated_content = compose_file.read_text() - assert "new_test_key" in updated_content - mock_fernet.generate_key.assert_called_once() + key = generate_fernet_key_string("TEST_KEY") + assert key == "NBJC_zYX6NWNek9v7tVv64YZz4K5sAgpoC4WGkQYv6I=" + default_key = generate_fernet_key_string() + assert default_key == "BMsag_V7iplH1SIxzrTIbhLRZYOAYd6p0_nPtGdmuxo=" diff --git a/task-sdk-integration-tests/docker-compose.yaml b/task-sdk-integration-tests/docker-compose.yaml index f2f2af37cdedc..e3b5e40f2970d 100644 --- a/task-sdk-integration-tests/docker-compose.yaml +++ b/task-sdk-integration-tests/docker-compose.yaml @@ -19,6 +19,8 @@ x-airflow-common: &airflow-common image: ${AIRFLOW_IMAGE_NAME} + env_file: + - ${ENV_FILE_PATH:-.env} environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: LocalExecutor @@ -26,7 +28,7 @@ x-airflow-common: AIRFLOW__CORE__AUTH_MANAGER: 'airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager' AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: 'true' AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow - AIRFLOW__CORE__FERNET_KEY: '' + AIRFLOW__CORE__FERNET_KEY: ${FERNET_KEY} AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__CORE__DAGS_FOLDER: '/opt/airflow/dags' AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' diff --git a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py index 19d410572e9cf..756473badb39c 100644 --- a/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py +++ b/task-sdk-integration-tests/tests/task_sdk_tests/conftest.py @@ -43,6 +43,8 @@ TASK_SDK_INTEGRATION_LOCAL_DOCKER_COMPOSE_FILE_PATH, ) +from tests_common.test_utils.fernet import generate_fernet_key_string + def print_diagnostics(compose, compose_version, docker_version): """Print diagnostic information when test fails.""" @@ -126,6 +128,12 @@ def docker_compose_setup(tmp_path_factory): print(f"AIRFLOW_IMAGE_NAME={DOCKER_IMAGE}", file=f) print(f"AIRFLOW_UID={os.getuid()}", file=f) print(f"HOST_OS={platform.system().lower()}", file=f) + # + # Please Do not use this Fernet key in any deployments! Please generate your own key. + # This is specifically generated for integration tests and not as default. + # + print(f"FERNET_KEY={generate_fernet_key_string()}", file=f) + docker_compose_files = [TASK_SDK_INTEGRATION_DOCKER_COMPOSE_FILE_PATH.as_posix()] log_level = "debug" if debugging_on else "info" if mount_volumes: