From 570faddaf0d5ed1e785b456421fb63abffbc53ac Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 7 Apr 2023 11:35:10 +0200 Subject: [PATCH 1/2] Enable AIP-44 and AIP-52 by default for development and CI on main The AIP-44 and AIP-52 are controlled now by environment variables, however those variables were not passed by default to inside the docker-compose environment so they had no effect when set on the ci.yaml. This PR fixes it, but it also sets the variables to be enabled by default in Breeze environment and when the tests are run locally in main using local venv so that the contributors are not surprised when they try to reproduce local failures. In 2.6 branch, we will set both variables to "false" by default in ci.yml, so that the tests are not run when we cherry-pick the changes. --- dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 2 ++ scripts/ci/docker-compose/_docker.env | 2 ++ scripts/ci/docker-compose/base.yml | 2 ++ scripts/ci/docker-compose/devcontainer.env | 2 ++ tests/conftest.py | 2 ++ 5 files changed, 10 insertions(+) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index ccc14c6bcd02e..88d8d59c83028 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -585,6 +585,8 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "AIRFLOW_CONSTRAINTS_MODE", "constraints-source-providers") set_value_to_default_if_not_set(env, "AIRFLOW_CONSTRAINTS_REFERENCE", "constraints-source-providers") set_value_to_default_if_not_set(env, "AIRFLOW_EXTRAS", "") + set_value_to_default_if_not_set(env, "AIRFLOW_ENABLE_AIP_44", "true") + set_value_to_default_if_not_set(env, "AIRFLOW_ENABLE_AIP_52", "true") set_value_to_default_if_not_set(env, "ANSWER", answer if answer is not None else "") set_value_to_default_if_not_set(env, "BASE_BRANCH", "main") set_value_to_default_if_not_set(env, "BREEZE", "true") diff --git a/scripts/ci/docker-compose/_docker.env b/scripts/ci/docker-compose/_docker.env index 3f465fe8ff7ec..4b9f30c4fef8f 100644 --- a/scripts/ci/docker-compose/_docker.env +++ b/scripts/ci/docker-compose/_docker.env @@ -16,6 +16,8 @@ # under the License. AIRFLOW_CI_IMAGE AIRFLOW_EXTRAS +AIRFLOW_ENABLE_AIP_44 +AIRFLOW_ENABLE_AIP_52 AIRFLOW_CONSTRAINTS_REFERENCE ANSWER BACKEND diff --git a/scripts/ci/docker-compose/base.yml b/scripts/ci/docker-compose/base.yml index b60b25850f1c1..a3d7d2ed1ac5b 100644 --- a/scripts/ci/docker-compose/base.yml +++ b/scripts/ci/docker-compose/base.yml @@ -29,6 +29,8 @@ services: # With env files and there are many problems with it: - AIRFLOW_CI_IMAGE=${AIRFLOW_CI_IMAGE} - AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS} + - AIRFLOW_ENABLE_AIP_44=${AIRFLOW_ENABLE_AIP_44} + - AIRFLOW_ENABLE_AIP_52=${AIRFLOW_ENABLE_AIP_52} - AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} - ANSWER=${ANSWER} - BACKEND=${BACKEND} diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index 482f8a0b9e7d0..cb78374219eac 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -17,6 +17,8 @@ HOME= AIRFLOW_CI_IMAGE="ghcr.io/apache/airflow/main/ci/python3.7:latest" ANSWER= +AIRFLOW_ENABLE_AIP_44="true" +AIRFLOW_ENABLE_AIP_44="true" PYTHON_MAJOR_MINOR_VERSION="3.7" AIRFLOW_EXTRAS= BASE_BRANCH="main" diff --git a/tests/conftest.py b/tests/conftest.py index 90892aca35910..cfae8468959a3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,8 @@ os.environ["AIRFLOW__CORE__UNIT_TEST_MODE"] = "True" os.environ["AWS_DEFAULT_REGION"] = os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" os.environ["CREDENTIALS_DIR"] = os.environ.get("CREDENTIALS_DIR") or "/files/airflow-breeze-config/keys" +os.environ["AIRFLOW_ENABLE_AIP_44"] = os.environ.get("AIRFLOW_ENABLE_AIP_44") or "true" +os.environ["AIRFLOW_ENABLE_AIP_52"] = os.environ.get("AIRFLOW_ENABLE_AIP_52") or "true" from airflow import settings # noqa: E402 from airflow.models.tasklog import LogTemplate # noqa: E402 From 07e2ae00ae506536783ad94c92b79a228bbcea5a Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 7 Apr 2023 16:06:51 +0200 Subject: [PATCH 2/2] Update scripts/ci/docker-compose/devcontainer.env --- scripts/ci/docker-compose/devcontainer.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/docker-compose/devcontainer.env b/scripts/ci/docker-compose/devcontainer.env index cb78374219eac..f39f6f0078a67 100644 --- a/scripts/ci/docker-compose/devcontainer.env +++ b/scripts/ci/docker-compose/devcontainer.env @@ -18,7 +18,7 @@ HOME= AIRFLOW_CI_IMAGE="ghcr.io/apache/airflow/main/ci/python3.7:latest" ANSWER= AIRFLOW_ENABLE_AIP_44="true" -AIRFLOW_ENABLE_AIP_44="true" +AIRFLOW_ENABLE_AIP_52="true" PYTHON_MAJOR_MINOR_VERSION="3.7" AIRFLOW_EXTRAS= BASE_BRANCH="main"