From 65b70158d164d1145531f9453a1d85209ebb6c9b Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 6 Nov 2024 17:54:49 +0000 Subject: [PATCH 1/2] Remove AIRFLOW_V_2_8_PLUS for provider's compatibility tests The AIRFLOW_V_2_8_PLUS is not needed as min airflow version already is 2.8.0+. --- contributing-docs/testing/unit_tests.rst | 4 +-- dev/README_RELEASE_PROVIDER_PACKAGES.md | 9 ++++-- .../aws/auth_manager/cli/test_avp_commands.py | 6 +--- .../aws/auth_manager/test_aws_auth_manager.py | 31 ++++++------------- .../operators/test_spark_kubernetes.py | 8 ----- .../tests/common/sql/hooks/test_dbapi.py | 6 ---- providers/tests/common/sql/hooks/test_sql.py | 5 --- .../tests/common/sql/hooks/test_sqlparse.py | 6 ---- .../tests/common/sql/operators/test_sql.py | 4 +-- .../common/sql/operators/test_sql_execute.py | 6 ---- providers/tests/common/sql/test_utils.py | 9 ------ .../tests/openlineage/plugins/test_utils.py | 3 +- tests_common/test_utils/compat.py | 1 - tests_common/test_utils/db.py | 5 +-- 14 files changed, 24 insertions(+), 79 deletions(-) diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index 2c136b55eb64a..976ff542e2ea0 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -1173,10 +1173,10 @@ are not part of the public API. We deal with it in one of the following ways: .. code-block:: python - from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS + from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS - @pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="The tests should be skipped for Airflow < 2.8") + @pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="The tests should be skipped for Airflow < 2.9") def some_test_that_only_works_for_airflow_2_8_plus(): pass diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 49631dd858764..f006bffd55597 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -87,13 +87,18 @@ You can read more about the command line tools used to generate the packages in the versions of Airflow that are not applicable anymore. 2. Check if Breeze unit tests in `dev/breeze/tests/test_packages.py` need adjustments. This is done by simply -searching and replacing old version occurrences with newer one. For example 2.5.0 to 2.6.0 +searching and replacing old version occurrences with newer one. For example 2.8.0 to 2.9.0 3. Update minimum airflow version for all packages, you should modify `MIN_AIRFLOW_VERSION` in `src/airflow_breeze/utils/packages.py` and run the `prepare-provider-documentation` command with the `--only-min-version-update` flag. This will only update the min version in the `__init__.py` files and package documentation without bumping the provider versions. +4. Remove `AIRFLOW_V_2_X_PLUS` in all tests (review and update skipif and other conditional + behaviour and test_compat.py, where X is the TARGET version we change to. For example + when we update min Airflow version to 2.9.0, we should remove all references to AIRFLOW_V_2_9_PLUS + simply because "everything" in our tests is already 2.9.0+ and there is no need to exclude or + modify tests for earlier versions of Airflow. Note: Sometimes we are releasing a subset of providers and would not want to add the list of these providers to every breeze command we run, specifically: @@ -111,7 +116,7 @@ branch="update-min-airflow-version" git checkout -b "${branch}" breeze release-management prepare-provider-documentation --only-min-version-update git add . -git commit -m "Bump minimum Airflow version in providers to Airflow 2.6.0" +git commit -m "Bump minimum Airflow version in providers to Airflow 2.9.0" git push --set-upstream origin "${branch}" ``` diff --git a/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py index faafd1ce497ea..d713d29115d5b 100644 --- a/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py +++ b/providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py @@ -24,15 +24,11 @@ from airflow.cli import cli_parser from airflow.providers.amazon.aws.auth_manager.cli.avp_commands import init_avp, update_schema -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS from tests_common.test_utils.config import conf_vars mock_boto3 = Mock() -pytestmark = [ - pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Test requires Airflow 2.8+"), - pytest.mark.skip_if_database_isolation_mode, -] +pytestmark = pytest.mark.skip_if_database_isolation_mode @pytest.mark.db_test diff --git a/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py index b62fa02889bce..acca912214802 100644 --- a/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py +++ b/providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py @@ -23,6 +23,15 @@ from flask import Flask, session from flask_appbuilder.menu import MenuItem +from airflow.auth.managers.models.resource_details import ( + AccessView, + ConfigurationDetails, + ConnectionDetails, + DagAccessEntity, + DagDetails, + PoolDetails, + VariableDetails, +) from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities from airflow.providers.amazon.aws.auth_manager.avp.facade import AwsAuthManagerAmazonVerifiedPermissionsFacade from airflow.providers.amazon.aws.auth_manager.aws_auth_manager import AwsAuthManager @@ -39,30 +48,10 @@ from airflow.www import app as application from airflow.www.extensions.init_appbuilder import init_appbuilder -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS from tests_common.test_utils.config import conf_vars from tests_common.test_utils.www import check_content_in_response -try: - from airflow.auth.managers.models.resource_details import ( - AccessView, - ConfigurationDetails, - ConnectionDetails, - DagAccessEntity, - DagDetails, - PoolDetails, - VariableDetails, - ) -except ImportError: - if not AIRFLOW_V_2_8_PLUS: - pytest.skip( - "Skipping tests that require airflow.auth.managers.models.resource_details for Airflow < 2.8.0", - allow_module_level=True, - ) - else: - raise - - if TYPE_CHECKING: from airflow.auth.managers.base_auth_manager import ResourceMethod from airflow.auth.managers.models.resource_details import AssetDetails diff --git a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py index 0b4c07be71d8c..39a649b613acf 100644 --- a/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py +++ b/providers/tests/cncf/kubernetes/operators/test_spark_kubernetes.py @@ -34,8 +34,6 @@ from airflow.utils import db, timezone from airflow.utils.types import DagRunType -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - @patch("airflow.providers.cncf.kubernetes.operators.spark_kubernetes.KubernetesHook") def test_spark_kubernetes_operator(mock_kubernetes_hook, data_file): @@ -780,9 +778,6 @@ def test_resolve_application_file_template_non_dictionary(dag_maker, tmp_path, b @pytest.mark.parametrize( "use_literal_value", [pytest.param(True, id="literal-value"), pytest.param(False, id="whitespace-compat")] ) -@pytest.mark.skipif( - not AIRFLOW_V_2_8_PLUS, reason="Skipping tests that require LiteralValue for Airflow < 2.8.0" -) def test_resolve_application_file_real_file( create_task_instance_of_operator, tmp_path, use_literal_value, session ): @@ -815,9 +810,6 @@ def test_resolve_application_file_real_file( @pytest.mark.db_test -@pytest.mark.skipif( - not AIRFLOW_V_2_8_PLUS, reason="Skipping tests that require LiteralValue for Airflow < 2.8.0" -) def test_resolve_application_file_real_file_not_exists(create_task_instance_of_operator, tmp_path, session): application_file = (tmp_path / "test-application-file.yml").resolve().as_posix() from airflow.template.templater import LiteralValue diff --git a/providers/tests/common/sql/hooks/test_dbapi.py b/providers/tests/common/sql/hooks/test_dbapi.py index a47b2856eb2fa..08bcf0b7fb0b9 100644 --- a/providers/tests/common/sql/hooks/test_dbapi.py +++ b/providers/tests/common/sql/hooks/test_dbapi.py @@ -30,12 +30,6 @@ from airflow.models import Connection from airflow.providers.common.sql.hooks.sql import DbApiHook, fetch_all_handler, fetch_one_handler -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - -pytestmark = [ - pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), -] - class DbApiHookInProvider(DbApiHook): conn_name_attr = "test_conn_id" diff --git a/providers/tests/common/sql/hooks/test_sql.py b/providers/tests/common/sql/hooks/test_sql.py index c1f919c704f72..669362ca56ee5 100644 --- a/providers/tests/common/sql/hooks/test_sql.py +++ b/providers/tests/common/sql/hooks/test_sql.py @@ -32,11 +32,6 @@ from airflow.utils.session import provide_session from providers.tests.common.sql.test_utils import mock_hook -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - -pytestmark = [ - pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), -] TASK_ID = "sql-operator" HOST = "host" diff --git a/providers/tests/common/sql/hooks/test_sqlparse.py b/providers/tests/common/sql/hooks/test_sqlparse.py index c8cca3aa35e81..72306287a4246 100644 --- a/providers/tests/common/sql/hooks/test_sqlparse.py +++ b/providers/tests/common/sql/hooks/test_sqlparse.py @@ -20,12 +20,6 @@ from airflow.providers.common.sql.hooks.sql import DbApiHook -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - -pytestmark = [ - pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), -] - @pytest.mark.parametrize( "line,parsed_statements", diff --git a/providers/tests/common/sql/operators/test_sql.py b/providers/tests/common/sql/operators/test_sql.py index 9ff5cebb76fa5..6274fa7ef747b 100644 --- a/providers/tests/common/sql/operators/test_sql.py +++ b/providers/tests/common/sql/operators/test_sql.py @@ -45,7 +45,7 @@ from airflow.utils.session import create_session from airflow.utils.state import State -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_3_0_PLUS +from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS from tests_common.test_utils.providers import get_provider_min_airflow_version if AIRFLOW_V_3_0_PLUS: @@ -53,7 +53,7 @@ pytestmark = [ pytest.mark.db_test, - pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), + pytest.mark.skipif(reason="Tests for Airflow 2.8.0+ only"), pytest.mark.skip_if_database_isolation_mode, ] diff --git a/providers/tests/common/sql/operators/test_sql_execute.py b/providers/tests/common/sql/operators/test_sql_execute.py index ee51dae1d5a32..095eb1fe82fcf 100644 --- a/providers/tests/common/sql/operators/test_sql_execute.py +++ b/providers/tests/common/sql/operators/test_sql_execute.py @@ -34,12 +34,6 @@ from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator from airflow.providers.openlineage.extractors.base import OperatorLineage -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - -pytestmark = [ - pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), -] - DATE = "2017-04-20" TASK_ID = "sql-operator" diff --git a/providers/tests/common/sql/test_utils.py b/providers/tests/common/sql/test_utils.py index 1c8625d1e9a9c..3e123ebf85b39 100644 --- a/providers/tests/common/sql/test_utils.py +++ b/providers/tests/common/sql/test_utils.py @@ -20,17 +20,8 @@ from typing import TYPE_CHECKING from unittest import mock -import pytest - from airflow.models import Connection -from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - -pytestmark = [ - pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="Tests for Airflow 2.8.0+ only"), -] - - if TYPE_CHECKING: from airflow.hooks.base import BaseHook diff --git a/providers/tests/openlineage/plugins/test_utils.py b/providers/tests/openlineage/plugins/test_utils.py index 531e21d42de1a..8c553a8d89537 100644 --- a/providers/tests/openlineage/plugins/test_utils.py +++ b/providers/tests/openlineage/plugins/test_utils.py @@ -48,7 +48,6 @@ from airflow.utils.state import State from tests_common.test_utils.compat import ( - AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS, AIRFLOW_V_2_10_PLUS, AIRFLOW_V_3_0_PLUS, @@ -428,7 +427,7 @@ def test_serialize_timetable_2_9(): @pytest.mark.skipif( - not AIRFLOW_V_2_8_PLUS or AIRFLOW_V_2_9_PLUS, + AIRFLOW_V_2_9_PLUS, reason="This test checks serialization only in 2.8 conditions", ) def test_serialize_timetable_2_8(): diff --git a/tests_common/test_utils/compat.py b/tests_common/test_utils/compat.py index 6f788c30dcca2..714cfd734577d 100644 --- a/tests_common/test_utils/compat.py +++ b/tests_common/test_utils/compat.py @@ -39,7 +39,6 @@ from airflow import __version__ as airflow_version AIRFLOW_VERSION = Version(airflow_version) -AIRFLOW_V_2_8_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("2.8.0") AIRFLOW_V_2_9_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("2.9.0") AIRFLOW_V_2_10_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("2.10.0") AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0") diff --git a/tests_common/test_utils/db.py b/tests_common/test_utils/db.py index d37a8e942e111..c9b62d096b936 100644 --- a/tests_common/test_utils/db.py +++ b/tests_common/test_utils/db.py @@ -62,16 +62,13 @@ def initial_db_init(): from airflow.www.extensions.init_appbuilder import init_appbuilder from airflow.www.extensions.init_auth_manager import get_auth_manager - from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS - db.resetdb() db.bootstrap_dagbag() # minimal app to add roles flask_app = Flask(__name__) flask_app.config["SQLALCHEMY_DATABASE_URI"] = conf.get("database", "SQL_ALCHEMY_CONN") init_appbuilder(flask_app) - if AIRFLOW_V_2_8_PLUS: - get_auth_manager().init() + get_auth_manager().init() def clear_db_runs(): From 1302bcc530bae702fe612c5705acc70f8bc0f84e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 6 Nov 2024 18:03:17 +0000 Subject: [PATCH 2/2] Update contributing-docs/testing/unit_tests.rst Co-authored-by: Vincent <97131062+vincbeck@users.noreply.github.com> --- contributing-docs/testing/unit_tests.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index 976ff542e2ea0..88ca07c6e0e68 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -1177,7 +1177,7 @@ are not part of the public API. We deal with it in one of the following ways: @pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="The tests should be skipped for Airflow < 2.9") - def some_test_that_only_works_for_airflow_2_8_plus(): + def some_test_that_only_works_for_airflow_2_9_plus(): pass 4) Sometimes, the tests should only be run when airflow is installed from the sources in main.