From ac5a3133f9f4e92e40c7fc7579d67b4dd462d3aa Mon Sep 17 00:00:00 2001 From: Mathieu Bouzard <13415583+mathbou@users.noreply.github.com> Date: Thu, 30 Mar 2023 12:20:04 +0200 Subject: [PATCH 1/7] set `_AIRFLOW__AS_LIBRARY="true"` for airflow version check --- When using `ExternalPythonOperator` in an Airflow instance with debug logging level, the airflow version check fails due to the presence of log output in the subprocess result --- airflow/operators/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/operators/python.py b/airflow/operators/python.py index 4158db2895c58..48a8eba6824e6 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -727,7 +727,7 @@ def _get_airflow_version_from_target_env(self) -> str | None: try: result = subprocess.check_output( - [self.python, "-c", "from airflow import __version__; print(__version__)"], text=True + [self.python, "-c", "from airflow import __version__; print(__version__)"], text=True, env=dict(os.environ, _AIRFLOW__AS_LIBRARY="true") ) target_airflow_version = result.strip() if target_airflow_version != airflow_version: From 45e82da53dd2fa73688ce8d7b67f5b301daf2486 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Fri, 31 Mar 2023 04:29:48 +0800 Subject: [PATCH 2/7] Fix static check errors --- airflow/operators/python.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/operators/python.py b/airflow/operators/python.py index 48a8eba6824e6..f5084e8e3f35b 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -727,7 +727,9 @@ def _get_airflow_version_from_target_env(self) -> str | None: try: result = subprocess.check_output( - [self.python, "-c", "from airflow import __version__; print(__version__)"], text=True, env=dict(os.environ, _AIRFLOW__AS_LIBRARY="true") + [self.python, "-c", "from airflow import __version__; print(__version__)"], + text=True, + env={**os.environ, "_AIRFLOW__AS_LIBRARY": "true"}, ) target_airflow_version = result.strip() if target_airflow_version != airflow_version: From fce0663ab72a092933772e07b87211e1ae50b28d Mon Sep 17 00:00:00 2001 From: Mathieu Bouzard <13415583+mathbou@users.noreply.github.com> Date: Thu, 13 Apr 2023 23:22:03 +0200 Subject: [PATCH 3/7] more explicit comments about _AIRFLOW__AS_LIBRARY --- airflow/__init__.py | 3 ++- airflow/operators/python.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airflow/__init__.py b/airflow/__init__.py index f0174e5d59f49..86ecc3077c4a9 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -61,7 +61,8 @@ # Perform side-effects unless someone has explicitly opted out before import -# WARNING: DO NOT USE THIS UNLESS YOU REALLY KNOW WHAT YOU'RE DOING. +# WARNING: Use with care, `_AIRFLOW__AS_LIBRARY` will prevent proper intialization (configs, syspath, logger...). +# It might be needed if you intend to use airflow as third-party package. if not os.environ.get("_AIRFLOW__AS_LIBRARY", None): settings.initialize() diff --git a/airflow/operators/python.py b/airflow/operators/python.py index f5084e8e3f35b..efdbd08aa464f 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -729,7 +729,7 @@ def _get_airflow_version_from_target_env(self) -> str | None: result = subprocess.check_output( [self.python, "-c", "from airflow import __version__; print(__version__)"], text=True, - env={**os.environ, "_AIRFLOW__AS_LIBRARY": "true"}, + env={**os.environ, "_AIRFLOW__AS_LIBRARY": "true"}, # Required to avoid `result` polluted by airflow init logs. ) target_airflow_version = result.strip() if target_airflow_version != airflow_version: From 097724adc7f931f3d0bab8efae7e4380db5343d3 Mon Sep 17 00:00:00 2001 From: Mathieu Bouzard <13415583+mathbou@users.noreply.github.com> Date: Fri, 14 Apr 2023 17:56:12 +0200 Subject: [PATCH 4/7] Update airflow/operators/python.py Co-authored-by: Tzu-ping Chung --- airflow/operators/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/operators/python.py b/airflow/operators/python.py index efdbd08aa464f..d33f5753c9203 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -729,7 +729,7 @@ def _get_airflow_version_from_target_env(self) -> str | None: result = subprocess.check_output( [self.python, "-c", "from airflow import __version__; print(__version__)"], text=True, - env={**os.environ, "_AIRFLOW__AS_LIBRARY": "true"}, # Required to avoid `result` polluted by airflow init logs. + env={**os.environ, "_AIRFLOW__AS_LIBRARY": "true"}, # Required to avoid `result` polluted by Airflow logs. ) target_airflow_version = result.strip() if target_airflow_version != airflow_version: From 5c57d4538ecbc163925d51fd835d3b3f368fc313 Mon Sep 17 00:00:00 2001 From: Mathieu Bouzard <13415583+mathbou@users.noreply.github.com> Date: Sun, 4 Jun 2023 15:21:16 +0200 Subject: [PATCH 5/7] Update airflow/__init__.py Co-authored-by: Tzu-ping Chung --- airflow/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airflow/__init__.py b/airflow/__init__.py index 86ecc3077c4a9..5f6850692f370 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -61,8 +61,10 @@ # Perform side-effects unless someone has explicitly opted out before import -# WARNING: Use with care, `_AIRFLOW__AS_LIBRARY` will prevent proper intialization (configs, syspath, logger...). -# It might be needed if you intend to use airflow as third-party package. +# WARNING: DO NOT USE THIS UNLESS YOU REALLY KNOW WHAT YOU'RE DOING. +# This environment variable prevents proper intialization, and things like +# configs, logging, the ORM, etc. will be broken. It is only useful if you only +# access certain trivial constants and free functions (e.g. `__version__`). if not os.environ.get("_AIRFLOW__AS_LIBRARY", None): settings.initialize() From 0e0bb6de0b6d9a64e3305260e09bc9b4e74063a3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 8 Jun 2023 15:09:40 +0800 Subject: [PATCH 6/7] Fix code format --- airflow/operators/python.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/operators/python.py b/airflow/operators/python.py index d33f5753c9203..23553e5a890a1 100644 --- a/airflow/operators/python.py +++ b/airflow/operators/python.py @@ -729,7 +729,8 @@ def _get_airflow_version_from_target_env(self) -> str | None: result = subprocess.check_output( [self.python, "-c", "from airflow import __version__; print(__version__)"], text=True, - env={**os.environ, "_AIRFLOW__AS_LIBRARY": "true"}, # Required to avoid `result` polluted by Airflow logs. + # Avoid Airflow logs polluting stdout. + env={**os.environ, "_AIRFLOW__AS_LIBRARY": "true"}, ) target_airflow_version = result.strip() if target_airflow_version != airflow_version: From 9bc0b88ca1254afb90eca7f01bd21764d5ab718d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 8 Jun 2023 19:41:54 +0200 Subject: [PATCH 7/7] Update airflow/__init__.py --- airflow/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/__init__.py b/airflow/__init__.py index 5f6850692f370..d330b9332a229 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -62,7 +62,7 @@ # Perform side-effects unless someone has explicitly opted out before import # WARNING: DO NOT USE THIS UNLESS YOU REALLY KNOW WHAT YOU'RE DOING. -# This environment variable prevents proper intialization, and things like +# This environment variable prevents proper initialization, and things like # configs, logging, the ORM, etc. will be broken. It is only useful if you only # access certain trivial constants and free functions (e.g. `__version__`). if not os.environ.get("_AIRFLOW__AS_LIBRARY", None):