From 944744bb642b8016c6edb585477d50f9ba459bc5 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Thu, 20 Mar 2025 13:31:16 +0000 Subject: [PATCH 1/2] (Re)move old dependencies from the old FAB UI Some of these deps, like python-nvd3 or markdown-it-py, are simply not used anymore, while others are still used and the dep has been moved to the fab provider where it is actually used. --- airflow-core/pyproject.toml | 22 ------ airflow-core/src/airflow/utils/json.py | 67 +------------------ airflow-core/tests/unit/utils/test_json.py | 25 ------- docs/spelling_wordlist.txt | 1 + generated/provider_dependencies.json | 8 ++- .../airflow/providers/amazon/aws/hooks/eks.py | 9 +-- .../amazon/aws/operators/sagemaker.py | 3 +- providers/fab/README.rst | 6 +- providers/fab/pyproject.toml | 16 ++++- .../providers/fab/get_provider_info.py | 6 +- 10 files changed, 37 insertions(+), 126 deletions(-) diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index 3ea64a6fe68d9..d3e46ce4106cd 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -69,12 +69,8 @@ dependencies = [ "argcomplete>=1.10", "asgiref>=2.3.0", "attrs>=22.1.0, !=25.2.0", - # Blinker use for signals in Flask, this is an optional dependency in Flask 2.2 and lower. - # In Flask 2.3 it becomes a mandatory dependency, and flask signals are always available. - "blinker>=1.6.2", "cadwyn>=5.1.3", "colorlog>=6.8.2", - "configupdater>=3.1.1", "cron-descriptor>=1.2.24", "croniter>=2.0.2", "cryptography>=41.0.0", @@ -86,15 +82,6 @@ dependencies = [ # 0.115.10 fastapi was a bad release that broke our API's and static checks. # Related fastapi issue here: https://github.com/fastapi/fastapi/discussions/13431 "fastapi[standard]>=0.112.4,!=0.115.10", - "flask-caching>=2.0.0", - # Flask-Session 0.6 add new arguments into the SqlAlchemySessionInterface constructor as well as - # all parameters now are mandatory which make AirflowDatabaseSessionInterface incompatible with this version. - "flask-session>=0.4.0,<0.6", - "flask-wtf>=1.1.0", - # Flask 2.3 is scheduled to introduce a number of deprecation removals - some of them might be breaking - # for our dependencies - notably `_app_ctx_stack` and `_request_ctx_stack` removals. - # We should remove the limitation after 2.3 is released and our dependencies are updated to handle it - "flask>=2.2.1,<2.3", "fsspec>=2023.10.0", "gitpython>=3.1.40", "gunicorn>=20.1.0", @@ -107,10 +94,7 @@ dependencies = [ "libcst >=1.1.0", "linkify-it-py>=2.0.0", "lockfile>=0.12.2", - "markdown-it-py>=2.1.0", - "markupsafe>=1.1.1", "marshmallow-oneofschema>=2.0.1", - "mdit-py-plugins>=0.3.0", "methodtools>=0.4.7", "opentelemetry-api>=1.24.0", "opentelemetry-exporter-otlp>=1.24.0", @@ -127,12 +111,9 @@ dependencies = [ "pyjwt>=2.10.0", "python-daemon>=3.0.0", "python-dateutil>=2.7.0", - "python-nvd3>=0.15.0", "python-slugify>=5.0", # Requests 3 if it will be released, will be heavily breaking. "requests>=2.27.0,<3", - "requests-toolbelt>=1.0.0", - "rfc3339-validator>=0.1.4", "rich-argparse>=1.0.0", "rich>=13.1.0", "setproctitle>=1.3.3", @@ -151,9 +132,6 @@ dependencies = [ # Does not work with it Tracked in https://github.com/fsspec/universal_pathlib/issues/276 "universal-pathlib>=0.2.2,!=0.2.4", "uuid6>=2024.7.10", - # Werkzug 3 breaks Flask-Login 0.6.2 - # we should remove this limitation when FAB supports Flask 2.3 - "werkzeug>=2.0,<4", ] diff --git a/airflow-core/src/airflow/utils/json.py b/airflow-core/src/airflow/utils/json.py index a8846282899f3..e7af172226f0c 100644 --- a/airflow-core/src/airflow/utils/json.py +++ b/airflow-core/src/airflow/utils/json.py @@ -18,70 +18,9 @@ from __future__ import annotations import json -from datetime import date, datetime -from decimal import Decimal from typing import Any -from flask.json.provider import JSONProvider - -from airflow.serialization.serde import CLASSNAME, DATA, SCHEMA_ID, deserialize, serialize -from airflow.utils.timezone import convert_to_utc, is_naive - - -class AirflowJsonProvider(JSONProvider): - """JSON Provider for Flask app to use WebEncoder.""" - - ensure_ascii: bool = True - sort_keys: bool = True - - def dumps(self, obj, **kwargs): - kwargs.setdefault("ensure_ascii", self.ensure_ascii) - kwargs.setdefault("sort_keys", self.sort_keys) - return json.dumps(obj, **kwargs, cls=WebEncoder) - - def loads(self, s: str | bytes, **kwargs): - return json.loads(s, **kwargs) - - -class WebEncoder(json.JSONEncoder): - """ - This encodes values into a web understandable format. There is no deserializer. - - This parses datetime, dates, Decimal and bytes. In order to parse the custom - classes and the other types, and since it's just to show the result in the UI, - we return repr(object) for everything else. - """ - - def default(self, o: Any) -> Any: - if isinstance(o, datetime): - if is_naive(o): - o = convert_to_utc(o) - return o.isoformat() - - if isinstance(o, date): - return o.strftime("%Y-%m-%d") - - if isinstance(o, Decimal): - data = serialize(o) - if isinstance(data, dict) and DATA in data: - return data[DATA] - if isinstance(o, bytes): - try: - return o.decode("unicode_escape") - except UnicodeDecodeError: - return repr(o) - try: - data = serialize(o) - if isinstance(data, dict) and CLASSNAME in data: - # this is here for backwards compatibility - if ( - data[CLASSNAME].startswith("numpy") - or data[CLASSNAME] == "kubernetes.client.models.v1_pod.V1Pod" - ): - return data[DATA] - return data - except TypeError: - return repr(o) +from airflow.serialization.serde import CLASSNAME, SCHEMA_ID, deserialize, serialize class XComEncoder(json.JSONEncoder): @@ -121,7 +60,3 @@ def object_hook(self, dct: dict) -> object: def orm_object_hook(dct: dict) -> object: """Create a readable representation of a serialized object.""" return deserialize(dct, False) - - -# backwards compatibility -AirflowJsonEncoder = WebEncoder diff --git a/airflow-core/tests/unit/utils/test_json.py b/airflow-core/tests/unit/utils/test_json.py index d5d0cdb32e8e2..2148830bd4372 100644 --- a/airflow-core/tests/unit/utils/test_json.py +++ b/airflow-core/tests/unit/utils/test_json.py @@ -19,11 +19,8 @@ import json from dataclasses import dataclass -from datetime import date, datetime from typing import ClassVar -import numpy as np -import pendulum import pytest from airflow.sdk.definitions.asset import Asset @@ -55,28 +52,6 @@ class U: x: int -class TestWebEncoder: - def test_encode_datetime(self): - obj = datetime.strptime("2017-05-21 00:00:00", "%Y-%m-%d %H:%M:%S") - assert json.dumps(obj, cls=utils_json.WebEncoder) == '"2017-05-21T00:00:00+00:00"' - - def test_encode_pendulum(self): - obj = pendulum.datetime(2017, 5, 21, tz="Asia/Kolkata") - assert json.dumps(obj, cls=utils_json.WebEncoder) == '"2017-05-21T00:00:00+05:30"' - - def test_encode_date(self): - assert json.dumps(date(2017, 5, 21), cls=utils_json.WebEncoder) == '"2017-05-21"' - - def test_encode_numpy_int(self): - assert json.dumps(np.int32(5), cls=utils_json.WebEncoder) == "5" - - def test_encode_numpy_bool(self): - assert json.dumps(np.bool_(True), cls=utils_json.WebEncoder) == "true" - - def test_encode_numpy_float(self): - assert json.dumps(np.float16(3.76953125), cls=utils_json.WebEncoder) == "3.76953125" - - class TestXComEncoder: def test_encode_raises(self): with pytest.raises(TypeError, match="^.*is not JSON serializable$"): diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 4076f14a09d42..b667f6c4fb7ac 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1175,6 +1175,7 @@ ntpd Nullable nullable num +nvd OAuth Oauth oauth diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 95e149de68276..5f765c9f10ede 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -594,11 +594,15 @@ "deps": [ "apache-airflow-providers-common-compat>=1.2.1", "apache-airflow>=3.0.0.dev0", + "blinker>=1.6.2", "connexion[flask]>=2.14.2,<3.0", "flask-appbuilder==4.5.3", "flask-login>=0.6.2", - "flask>=2.2,<2.3", - "jmespath>=0.7.0" + "flask-session>=0.4.0,<0.6", + "flask-wtf>=1.1.0", + "flask>=2.2.1,<2.3", + "jmespath>=0.7.0", + "werkzeug>=2.2,<4" ], "devel-deps": [ "kerberos>=1.3.0" diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/eks.py b/providers/amazon/src/airflow/providers/amazon/aws/hooks/eks.py index 421e66b606a26..60d3b5bb7a078 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/eks.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/eks.py @@ -35,7 +35,6 @@ from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook from airflow.providers.amazon.aws.hooks.sts import StsHook from airflow.utils import yaml -from airflow.utils.json import AirflowJsonEncoder DEFAULT_PAGINATION_TOKEN = "" STS_TOKEN_EXPIRES_IN = 60 @@ -315,7 +314,7 @@ def describe_cluster(self, name: str, verbose: bool = False) -> dict: ) if verbose: cluster_data = response.get("cluster") - self.log.info("Amazon EKS cluster details: %s", json.dumps(cluster_data, cls=AirflowJsonEncoder)) + self.log.info("Amazon EKS cluster details: %s", json.dumps(cluster_data, default=repr)) return response def describe_nodegroup(self, clusterName: str, nodegroupName: str, verbose: bool = False) -> dict: @@ -343,7 +342,7 @@ def describe_nodegroup(self, clusterName: str, nodegroupName: str, verbose: bool nodegroup_data = response.get("nodegroup") self.log.info( "Amazon EKS managed node group details: %s", - json.dumps(nodegroup_data, cls=AirflowJsonEncoder), + json.dumps(nodegroup_data, default=repr), ) return response @@ -374,9 +373,7 @@ def describe_fargate_profile( ) if verbose: fargate_profile_data = response.get("fargateProfile") - self.log.info( - "AWS Fargate profile details: %s", json.dumps(fargate_profile_data, cls=AirflowJsonEncoder) - ) + self.log.info("AWS Fargate profile details: %s", json.dumps(fargate_profile_data, default=repr)) return response def get_cluster_state(self, clusterName: str) -> ClusterStates: diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py index dc68f13bb36b5..c3d74acd7bfc8 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py @@ -44,7 +44,6 @@ from airflow.providers.amazon.aws.utils.sagemaker import ApprovalStatus from airflow.providers.amazon.aws.utils.tags import format_tags from airflow.utils.helpers import prune_dict -from airflow.utils.json import AirflowJsonEncoder if TYPE_CHECKING: from airflow.providers.common.compat.openlineage.facet import Dataset @@ -56,7 +55,7 @@ def serialize(result: dict) -> dict: - return json.loads(json.dumps(result, cls=AirflowJsonEncoder)) + return json.loads(json.dumps(result, default=repr)) class SageMakerBaseOperator(BaseOperator): diff --git a/providers/fab/README.rst b/providers/fab/README.rst index f04c442fd1097..f54d341f885c1 100644 --- a/providers/fab/README.rst +++ b/providers/fab/README.rst @@ -55,11 +55,15 @@ PIP package Version required ========================================== ================== ``apache-airflow`` ``>=3.0.0.dev0`` ``apache-airflow-providers-common-compat`` ``>=1.2.1`` -``flask`` ``>=2.2,<2.3`` +``blinker`` ``>=1.6.2`` +``flask`` ``>=2.2.1,<2.3`` ``flask-appbuilder`` ``==4.5.3`` ``flask-login`` ``>=0.6.2`` +``flask-session`` ``>=0.4.0,<0.6`` +``flask-wtf`` ``>=1.1.0`` ``connexion[flask]`` ``>=2.14.2,<3.0`` ``jmespath`` ``>=0.7.0`` +``werkzeug`` ``>=2.2,<4`` ========================================== ================== Cross provider package dependencies diff --git a/providers/fab/pyproject.toml b/providers/fab/pyproject.toml index 78f03450f38bc..8c97d3fc9ce01 100644 --- a/providers/fab/pyproject.toml +++ b/providers/fab/pyproject.toml @@ -59,7 +59,13 @@ requires-python = "~=3.9" dependencies = [ "apache-airflow>=3.0.0.dev0", "apache-airflow-providers-common-compat>=1.2.1", - "flask>=2.2,<2.3", + # Blinker use for signals in Flask, this is an optional dependency in Flask 2.2 and lower. + # In Flask 2.3 it becomes a mandatory dependency, and flask signals are always available. + "blinker>=1.6.2", + # Flask 2.3 is scheduled to introduce a number of deprecation removals - some of them might be breaking + # for our dependencies - notably `_app_ctx_stack` and `_request_ctx_stack` removals. + # We should remove the limitation after 2.3 is released and our dependencies are updated to handle it + "flask>=2.2.1,<2.3", # We are tightly coupled with FAB version as we vendored-in part of FAB code related to security manager # This is done as part of preparation to removing FAB as dependency, but we are not ready for it yet # Every time we update FAB version here, please make sure that you review the classes and models in @@ -67,8 +73,16 @@ dependencies = [ # In particular, make sure any breaking changes, for example any new methods, are accounted for. "flask-appbuilder==4.5.3", "flask-login>=0.6.2", + # Flask-Session 0.6 add new arguments into the SqlAlchemySessionInterface constructor as well as + # all parameters now are mandatory which make AirflowDatabaseSessionInterface incompatible with this version. + "flask-session>=0.4.0,<0.6", + "flask-wtf>=1.1.0", "connexion[flask]>=2.14.2,<3.0", "jmespath>=0.7.0", + # Werkzug 3 breaks Flask-Login 0.6.2 + # we should remove this limitation when FAB supports Flask 2.3 + "werkzeug>=2.2,<4", + ] # The optional dependencies should be modified in place in the generated file diff --git a/providers/fab/src/airflow/providers/fab/get_provider_info.py b/providers/fab/src/airflow/providers/fab/get_provider_info.py index 026a8325f0349..a6dc998cfc4db 100644 --- a/providers/fab/src/airflow/providers/fab/get_provider_info.py +++ b/providers/fab/src/airflow/providers/fab/get_provider_info.py @@ -86,11 +86,15 @@ def get_provider_info(): "dependencies": [ "apache-airflow>=3.0.0.dev0", "apache-airflow-providers-common-compat>=1.2.1", - "flask>=2.2,<2.3", + "blinker>=1.6.2", + "flask>=2.2.1,<2.3", "flask-appbuilder==4.5.3", "flask-login>=0.6.2", + "flask-session>=0.4.0,<0.6", + "flask-wtf>=1.1.0", "connexion[flask]>=2.14.2,<3.0", "jmespath>=0.7.0", + "werkzeug>=2.2,<4", ], "optional-dependencies": {"kerberos": ["kerberos>=1.3.0"]}, "devel-dependencies": ["kerberos>=1.3.0"], From 3148d604e8f7c500ac758ec0c53f44ab470ee16c Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Mon, 24 Mar 2025 13:44:42 +0000 Subject: [PATCH 2/2] fixup! (Re)move old dependencies from the old FAB UI --- RELEASE_NOTES.rst | 6 ++++++ docs/spelling_wordlist.txt | 1 - reproducible_build.yaml | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 780e33ee8e4af..756da7050db20 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -14277,3 +14277,9 @@ Airflow 1.7.1 (2016-05-19) - Use GSSAPI instead of KERBEROS and provide backwards compatibility - ISSUE-1123 Use impyla instead of pyhs2 - Set celery_executor to use queue name as exchange + + +.. spelling:: + + nvd + lineChart diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index b667f6c4fb7ac..4076f14a09d42 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -1175,7 +1175,6 @@ ntpd Nullable nullable num -nvd OAuth Oauth oauth diff --git a/reproducible_build.yaml b/reproducible_build.yaml index 00ac3b9fa5e42..dfe504f7dddea 100644 --- a/reproducible_build.yaml +++ b/reproducible_build.yaml @@ -1,2 +1,2 @@ -release-notes-hash: 8c02811625ac7262d5ce34eb4cd7c5b0 -source-date-epoch: 1742481556 +release-notes-hash: 0aba23cb51bcf2ed752e8591c8f11e0f +source-date-epoch: 1742823847