From d3642543656f942699c5a21f797c24ba2717af50 Mon Sep 17 00:00:00 2001 From: Vincent Beck Date: Wed, 23 Aug 2023 16:55:36 -0400 Subject: [PATCH 1/2] Fix bug in task logs when using AWS CloudWatch. Do not set `start_time` --- .../amazon/aws/log/cloudwatch_task_handler.py | 4 ---- .../aws/log/test_cloudwatch_task_handler.py | 19 +++---------------- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py b/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py index 5f74468d04ce4..0cc9c43b05cbf 100644 --- a/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +++ b/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py @@ -114,9 +114,6 @@ def get_cloudwatch_logs(self, stream_name: str, task_instance: TaskInstance) -> :param task_instance: the task instance to get logs about :return: string of all logs from the given log stream """ - start_time = ( - 0 if task_instance.start_date is None else datetime_to_epoch_utc_ms(task_instance.start_date) - ) # If there is an end_date to the task instance, fetch logs until that date + 30 seconds # 30 seconds is an arbitrary buffer so that we don't miss any logs that were emitted end_time = ( @@ -127,7 +124,6 @@ def get_cloudwatch_logs(self, stream_name: str, task_instance: TaskInstance) -> events = self.hook.get_log_events( log_group=self.log_group, log_stream_name=stream_name, - start_time=start_time, end_time=end_time, ) return "\n".join(self._event_to_str(event) for event in events) diff --git a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py b/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py index 219f594604a4a..66da7490ed116 100644 --- a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py +++ b/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py @@ -155,29 +155,16 @@ def test_read(self): ) @pytest.mark.parametrize( - "start_date, end_date, expected_start_time, expected_end_time", + "end_date, expected_end_time", [ - (None, None, 0, None), - (datetime(2020, 1, 1), None, datetime_to_epoch_utc_ms(datetime(2020, 1, 1)), None), - ( - None, - datetime(2020, 1, 2), - 0, - datetime_to_epoch_utc_ms(datetime(2020, 1, 2) + timedelta(seconds=30)), - ), - ( - datetime(2020, 1, 1), - datetime(2020, 1, 2), - datetime_to_epoch_utc_ms(datetime(2020, 1, 1)), - datetime_to_epoch_utc_ms(datetime(2020, 1, 2) + timedelta(seconds=30)), - ), + (None, None), + (datetime(2020, 1, 2), datetime_to_epoch_utc_ms(datetime(2020, 1, 2) + timedelta(seconds=30))), ], ) @mock.patch.object(AwsLogsHook, "get_log_events") def test_get_cloudwatch_logs( self, mock_get_log_events, start_date, end_date, expected_start_time, expected_end_time ): - self.ti.start_date = start_date self.ti.end_date = end_date self.cloudwatch_task_handler.get_cloudwatch_logs(self.remote_log_stream, self.ti) mock_get_log_events.assert_called_once_with( From 7b7733935ea8bf3d426b91fa74c1b02184b6c567 Mon Sep 17 00:00:00 2001 From: Vincent Beck Date: Wed, 23 Aug 2023 17:33:25 -0400 Subject: [PATCH 2/2] Fix tests --- .../providers/amazon/aws/log/test_cloudwatch_task_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py b/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py index 66da7490ed116..00d5cf2f46493 100644 --- a/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py +++ b/tests/providers/amazon/aws/log/test_cloudwatch_task_handler.py @@ -162,15 +162,12 @@ def test_read(self): ], ) @mock.patch.object(AwsLogsHook, "get_log_events") - def test_get_cloudwatch_logs( - self, mock_get_log_events, start_date, end_date, expected_start_time, expected_end_time - ): + def test_get_cloudwatch_logs(self, mock_get_log_events, end_date, expected_end_time): self.ti.end_date = end_date self.cloudwatch_task_handler.get_cloudwatch_logs(self.remote_log_stream, self.ti) mock_get_log_events.assert_called_once_with( log_group=self.remote_log_group, log_stream_name=self.remote_log_stream, - start_time=expected_start_time, end_time=expected_end_time, )