From 1246435dfe17202337f776696cc9c39bb6db5ea2 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Tue, 3 Jun 2025 14:25:20 +0530 Subject: [PATCH 1/2] Allow migration to pytest 8.4.0 --- dev/breeze/README.md | 2 +- dev/breeze/pyproject.toml | 4 +--- devel-common/pyproject.toml | 4 +--- .../amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py | 3 ++- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 6188e718aba3e..7b2cae038eb81 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -135,6 +135,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: 14ae9299422a78b0af9db9ec4e81c9b6a9412f93a5499a367da252cf58d13e1c8bae31d3642c693e356136fb07a94c25569d088acee5521099a08274dcae3278 +Package config hash: 6c1d151d53432393683e6564db05ebd9561946786332ca15f2dff72e5307d8c651c333ce3a6d305b97516376e2485f0a309d2f8f858437f4df68694ed7d4cd8b --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml index 52acecef6b738..f0137d2ce222c 100644 --- a/dev/breeze/pyproject.toml +++ b/dev/breeze/pyproject.toml @@ -65,9 +65,7 @@ dependencies = [ "psutil>=5.9.6", "pygithub>=2.1.1", "pytest-xdist>=3.3.1", - # 8.4.0 introduces changes related to fixture that cause CI failure. - # TODO: we'll need to check how to get around with it. - "pytest>=8.2,<8.4.0", + "pytest>=8.2", "pyyaml>=6.0.2", "requests>=2.31.0", "restructuredtext-lint>=1.4.0", diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml index 9cd7d3b829ae5..d8f69f320e8f7 100644 --- a/devel-common/pyproject.toml +++ b/devel-common/pyproject.toml @@ -137,9 +137,7 @@ dependencies = [ "pytest-timeouts>=1.2.1", "pytest-unordered>=0.6.1", "pytest-xdist>=3.5.0", - # 8.4.0 introduces changes related to fixture that cause CI failure. - # TODO: we'll need to check how to get around with it. - "pytest>=8.3.3,<8.4.0", + "pytest>=8.3.3", ] "sentry" = [ "blinker>=1.7.0", diff --git a/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py b/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py index d67b25d07f2a6..e7ae40d7a3a73 100644 --- a/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py +++ b/providers/amazon/tests/unit/amazon/aws/hooks/test_batch_waiters.py @@ -45,7 +45,6 @@ def aws_region(): return AWS_REGION -@mock_aws @pytest.fixture def patch_hook(monkeypatch, aws_region): """Patch hook object by dummy boto3 Batch client.""" @@ -59,6 +58,7 @@ def test_batch_waiters(aws_region): assert isinstance(batch_waiters, BatchWaitersHook) +@mock_aws class TestBatchWaiters: @pytest.fixture(autouse=True) def setup_tests(self, patch_hook): @@ -215,6 +215,7 @@ def test_wait_for_job_raises_for_waiter_error(self): assert mock_waiter.wait.call_count == 1 +@mock_aws class TestBatchJobWaiters: """Test default waiters.""" From bc7dea8fb6b85e088b59415b37bf02abc7e5c57d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 3 Jun 2025 11:09:56 +0200 Subject: [PATCH 2/2] fixup! Allow migration to pytest 8.4.0 --- dev/breeze/README.md | 2 +- dev/breeze/pyproject.toml | 2 +- .../aws/log/test_cloudwatch_task_handler.py | 70 +++++++++---------- .../amazon/aws/log/test_s3_task_handler.py | 62 ++++++++-------- 4 files changed, 68 insertions(+), 68 deletions(-) diff --git a/dev/breeze/README.md b/dev/breeze/README.md index 7b2cae038eb81..6c6eba5e9993a 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -135,6 +135,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: 6c1d151d53432393683e6564db05ebd9561946786332ca15f2dff72e5307d8c651c333ce3a6d305b97516376e2485f0a309d2f8f858437f4df68694ed7d4cd8b +Package config hash: 19b7a69c4b7ef23d1c665286fd7ca1a1d8c28fa9ba8523da6c3e215d8cd7c4bc0406186898a90c92d8e9f527bc8fa8d5c6407f914d7674e59e4981bb3c795e8c --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/pyproject.toml b/dev/breeze/pyproject.toml index f0137d2ce222c..eb0b1b49574f7 100644 --- a/dev/breeze/pyproject.toml +++ b/dev/breeze/pyproject.toml @@ -65,7 +65,7 @@ dependencies = [ "psutil>=5.9.6", "pygithub>=2.1.1", "pytest-xdist>=3.3.1", - "pytest>=8.2", + "pytest>=8.3.3", "pyyaml>=6.0.2", "requests>=2.31.0", "restructuredtext-lint>=1.4.0", diff --git a/providers/amazon/tests/unit/amazon/aws/log/test_cloudwatch_task_handler.py b/providers/amazon/tests/unit/amazon/aws/log/test_cloudwatch_task_handler.py index a528014328590..30a66e46608d4 100644 --- a/providers/amazon/tests/unit/amazon/aws/log/test_cloudwatch_task_handler.py +++ b/providers/amazon/tests/unit/amazon/aws/log/test_cloudwatch_task_handler.py @@ -184,43 +184,43 @@ def test_event_to_str(self): @pytest.mark.db_test class TestCloudwatchTaskHandler: - @conf_vars({("logging", "remote_log_conn_id"): "aws_default"}) @pytest.fixture(autouse=True) - def setup_tests(self, create_log_template, tmp_path_factory, session): - self.remote_log_group = "log_group_name" - self.region_name = "us-west-2" - self.local_log_location = str(tmp_path_factory.mktemp("local-cloudwatch-log-location")) - if AIRFLOW_V_3_0_PLUS: - create_log_template("{dag_id}/{task_id}/{logical_date}/{try_number}.log") - else: - create_log_template("{dag_id}/{task_id}/{execution_date}/{try_number}.log") - self.cloudwatch_task_handler = CloudwatchTaskHandler( - self.local_log_location, - f"arn:aws:logs:{self.region_name}:11111111:log-group:{self.remote_log_group}", - ) - - date = datetime(2020, 1, 1) - dag_id = "dag_for_testing_cloudwatch_task_handler" - task_id = "task_for_testing_cloudwatch_log_handler" - self.dag = DAG(dag_id=dag_id, schedule=None, start_date=date) - task = EmptyOperator(task_id=task_id, dag=self.dag) - if AIRFLOW_V_3_0_PLUS: - dag_run = DagRun( - dag_id=self.dag.dag_id, - logical_date=date, - run_id="test", - run_type="scheduled", - ) - else: - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=date, - run_id="test", - run_type="scheduled", + def setup(self, create_log_template, tmp_path_factory, session): + with conf_vars({("logging", "remote_log_conn_id"): "aws_default"}): + self.remote_log_group = "log_group_name" + self.region_name = "us-west-2" + self.local_log_location = str(tmp_path_factory.mktemp("local-cloudwatch-log-location")) + if AIRFLOW_V_3_0_PLUS: + create_log_template("{dag_id}/{task_id}/{logical_date}/{try_number}.log") + else: + create_log_template("{dag_id}/{task_id}/{execution_date}/{try_number}.log") + self.cloudwatch_task_handler = CloudwatchTaskHandler( + self.local_log_location, + f"arn:aws:logs:{self.region_name}:11111111:log-group:{self.remote_log_group}", ) - session.add(dag_run) - session.commit() - session.refresh(dag_run) + + date = datetime(2020, 1, 1) + dag_id = "dag_for_testing_cloudwatch_task_handler" + task_id = "task_for_testing_cloudwatch_log_handler" + self.dag = DAG(dag_id=dag_id, schedule=None, start_date=date) + task = EmptyOperator(task_id=task_id, dag=self.dag) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=date, + run_id="test", + run_type="scheduled", + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=date, + run_id="test", + run_type="scheduled", + ) + session.add(dag_run) + session.commit() + session.refresh(dag_run) self.ti = TaskInstance(task=task, run_id=dag_run.run_id) self.ti.dag_run = dag_run diff --git a/providers/amazon/tests/unit/amazon/aws/log/test_s3_task_handler.py b/providers/amazon/tests/unit/amazon/aws/log/test_s3_task_handler.py index 45a39e33ad4e8..29246f5d23054 100644 --- a/providers/amazon/tests/unit/amazon/aws/log/test_s3_task_handler.py +++ b/providers/amazon/tests/unit/amazon/aws/log/test_s3_task_handler.py @@ -46,39 +46,39 @@ def s3mock(): @pytest.mark.db_test class TestS3RemoteLogIO: - @conf_vars({("logging", "remote_log_conn_id"): "aws_default"}) @pytest.fixture(autouse=True) def setup_tests(self, create_log_template, tmp_path_factory, session): - self.remote_log_base = "s3://bucket/remote/log/location" - self.remote_log_location = "s3://bucket/remote/log/location/1.log" - self.remote_log_key = "remote/log/location/1.log" - self.local_log_location = str(tmp_path_factory.mktemp("local-s3-log-location")) - create_log_template("{try_number}.log") - self.s3_task_handler = S3TaskHandler(self.local_log_location, self.remote_log_base) - # Verify the hook now with the config override - self.subject = self.s3_task_handler.io - assert self.subject.hook is not None - - date = datetime(2016, 1, 1) - self.dag = DAG("dag_for_testing_s3_task_handler", schedule=None, start_date=date) - task = EmptyOperator(task_id="task_for_testing_s3_log_handler", dag=self.dag) - if AIRFLOW_V_3_0_PLUS: - dag_run = DagRun( - dag_id=self.dag.dag_id, - logical_date=date, - run_id="test", - run_type="manual", - ) - else: - dag_run = DagRun( - dag_id=self.dag.dag_id, - execution_date=date, - run_id="test", - run_type="manual", - ) - session.add(dag_run) - session.commit() - session.refresh(dag_run) + with conf_vars({("logging", "remote_log_conn_id"): "aws_default"}): + self.remote_log_base = "s3://bucket/remote/log/location" + self.remote_log_location = "s3://bucket/remote/log/location/1.log" + self.remote_log_key = "remote/log/location/1.log" + self.local_log_location = str(tmp_path_factory.mktemp("local-s3-log-location")) + create_log_template("{try_number}.log") + self.s3_task_handler = S3TaskHandler(self.local_log_location, self.remote_log_base) + # Verify the hook now with the config override + self.subject = self.s3_task_handler.io + assert self.subject.hook is not None + + date = datetime(2016, 1, 1) + self.dag = DAG("dag_for_testing_s3_task_handler", schedule=None, start_date=date) + task = EmptyOperator(task_id="task_for_testing_s3_log_handler", dag=self.dag) + if AIRFLOW_V_3_0_PLUS: + dag_run = DagRun( + dag_id=self.dag.dag_id, + logical_date=date, + run_id="test", + run_type="manual", + ) + else: + dag_run = DagRun( + dag_id=self.dag.dag_id, + execution_date=date, + run_id="test", + run_type="manual", + ) + session.add(dag_run) + session.commit() + session.refresh(dag_run) self.ti = TaskInstance(task=task, run_id=dag_run.run_id) self.ti.dag_run = dag_run