From f95b641c6259190c957a7ef2c9ec3e4c9c8f8425 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Wed, 14 Jun 2023 21:00:05 -0700 Subject: [PATCH 01/23] Create ChimeWebhook Hook to send messages to chime room from Airflow. --- airflow/providers/amazon/aws/hooks/chime.py | 104 +++++++++++++++++ airflow/providers/amazon/provider.yaml | 5 + generated/provider_dependencies.json | 1 + .../providers/amazon/aws/hooks/test_chime.py | 106 ++++++++++++++++++ 4 files changed, 216 insertions(+) create mode 100644 airflow/providers/amazon/aws/hooks/chime.py create mode 100644 tests/providers/amazon/aws/hooks/test_chime.py diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py new file mode 100644 index 0000000000000..1c11d07d45e79 --- /dev/null +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -0,0 +1,104 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""This module contains a web hook for Chime.""" +from __future__ import annotations + +import json +import re +from typing import Any + +from airflow.exceptions import AirflowException +from airflow.providers.http.hooks.http import HttpHook + + +class ChimeWebhookHook(HttpHook): + """Interact with Chime Web Hooks to create notifications. + + .. warning:: This hook is only designed to work with Web Hooks and not Chatbots. + + :param http_conn_id: Http connection ID with host as "https://hooks.chime.aws" and + default webhook endpoint in the extra field in the form of + {"webhook_endpoint": "incomingwebhooks/{webhook.id}?token{webhook.token}"} + :param webhook_endpoint: Discord webhook endpoint in the form of + "incomingwebhooks/{webhook.id}?token={webhook.token}" + :param message: The message you want to send to your Discord channel + (max 4096 characters) + """ + + conn_name_attr = "http_conn_id" + default_conn_name = "chime_default" + conn_type = "chime" + hook_name = "Chime Web Hook" + + def __init__( + self, + http_conn_id: str | None = None, + webhook_endpoint: str | None = None, + message: str = "", + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self.message = message + self.webhook_endpoint = self._get_webhook_endpoint(http_conn_id, webhook_endpoint) + + def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str | None) -> str: + """ + Given a Chime http_conn_id return the default webhook endpoint or override if + webhook_endpoint is manually provided. + + :param http_conn_id: The provided connection ID + :param webhook_endpoint: The manually provided webhook endpoint + :return: Webhook Endpoint(str) to use with Chime + """ + if webhook_endpoint: + endpoint = webhook_endpoint + elif http_conn_id: + conn = self.get_connection(http_conn_id) + extra = conn.extra_dejson + endpoint = extra.get("webhook_endpoint", "") + else: + raise AirflowException() + + # Check to make sure the endpoint matches what Chime expects + if not re.match("^incomingwebhooks/[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", endpoint): + raise AirflowException( + "Expected Chime webhook endpoint in the form of " + '"incomingwebhooks/{webhook.id}?token={webhook.token}".' + ) + + return endpoint + + def _build_chime_payload(self) -> str: + """Builds payload for Chime and ensures messages do not exceed max length allowed.""" + payload: dict[str, Any] = {} + # We need to make sure that the message does not exceed the max length for Chime + if len(self.message) <= 4096: + payload["Content"] = self.message + else: + raise AirflowException("Chime message must be 4096 characters or less.") + + return json.dumps(payload) + + def execute(self) -> None: + """Execute calling the Chime webhook endpoint.""" + chime_payload = self._build_chime_payload() + self.run( + endpoint=self.webhook_endpoint, data=chime_payload, headers={"Content-type": "application/json"} + ) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 51af2e4d24673..1c138a495cc1f 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -406,6 +406,9 @@ hooks: - integration-name: Amazon Athena python-modules: - airflow.providers.amazon.aws.hooks.athena + - integration-name: Amazon Chime + python-modules: + - airflow.providers.amazon.aws.hooks.chime - integration-name: Amazon DynamoDB python-modules: - airflow.providers.amazon.aws.hooks.dynamodb @@ -624,6 +627,8 @@ connection-types: connection-type: emr - hook-class-name: airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook connection-type: redshift + - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook + connection-type: chime secrets-backends: - airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index d77fd32e8d46d..50d69b309ff65 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -40,6 +40,7 @@ "exasol", "ftp", "google", + "http", "imap", "mongo", "salesforce", diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/tests/providers/amazon/aws/hooks/test_chime.py new file mode 100644 index 0000000000000..fafe9c5e69d92 --- /dev/null +++ b/tests/providers/amazon/aws/hooks/test_chime.py @@ -0,0 +1,106 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json + +import pytest + +from airflow.exceptions import AirflowException +from airflow.models import Connection +from airflow.providers.amazon.aws.hooks.chime import ChimeWebhookHook +from airflow.utils import db + + +class TestChimeWebhookHook: + + _config = { + "http_conn_id": "default-chime-webhook", + "webhook_endpoint": "incomingwebhooks/abcd-1134?token=somechimetoken_111", + "message": "your message here", + } + + expected_payload_dict = { + "Content": _config["message"], + } + + expected_payload = json.dumps(expected_payload_dict) + + def setup_method(self): + db.merge_conn( + Connection( + conn_id="default-chime-webhook", + conn_type="chime", + host="https://hooks.chime.aws", + extra='{"webhook_endpoint": "incomingwebhooks/abcd-1134?token=somechimetoken_111"}', + ) + ) + + def test_get_webhook_endpoint_manual_token(self): + # Given + provided_endpoint = "incomingwebhooks/abcd-1134?token=somechimetoken_111" + hook = ChimeWebhookHook(webhook_endpoint=provided_endpoint) + + # When + webhook_endpoint = hook._get_webhook_endpoint(None, provided_endpoint) + + # Then + assert webhook_endpoint == provided_endpoint + + def test_get_webhook_endpoint_invalid_url(self): + # Given + provided_endpoint = "https://hooks.chime.aws/some-invalid-webhook-url" + + # When/Then + expected_message = "Expected Chime webhook endpoint in the form of" + with pytest.raises(AirflowException, match=expected_message): + ChimeWebhookHook(webhook_endpoint=provided_endpoint) + + def test_get_webhook_endpoint_conn_id(self): + # Given + conn_id = "default-chime-webhook" + hook = ChimeWebhookHook(http_conn_id=conn_id) + expected_webhook_endpoint = "incomingwebhooks/abcd-1134?token=somechimetoken_111" + + # When + webhook_endpoint = hook._get_webhook_endpoint(conn_id, None) + + # Then + assert webhook_endpoint == expected_webhook_endpoint + + def test_build_chime_payload(self): + # Given + hook = ChimeWebhookHook(**self._config) + + # When + payload = hook._build_chime_payload() + + # Then + assert self.expected_payload == payload + + def test_build_chime_payload_message_length(self): + # Given + config = self._config.copy() + # create message over the character limit + config["message"] = "c" * 4097 + hook = ChimeWebhookHook(**config) + + # When/Then + expected_message = "Chime message must be 4096 characters or less." + with pytest.raises(AirflowException, match=expected_message): + hook._build_chime_payload() From b7ca461494827b6da0d468fa279bfca7f67ebf58 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Thu, 15 Jun 2023 15:24:29 -0700 Subject: [PATCH 02/23] Fix method name/Add connection docs --- airflow/providers/amazon/aws/hooks/chime.py | 20 ++--- .../connections/chime.rst | 73 +++++++++++++++++++ 2 files changed, 83 insertions(+), 10 deletions(-) create mode 100644 docs/apache-airflow-providers-amazon/connections/chime.rst diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index 1c11d07d45e79..aaba80f2399ab 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -35,9 +35,9 @@ class ChimeWebhookHook(HttpHook): :param http_conn_id: Http connection ID with host as "https://hooks.chime.aws" and default webhook endpoint in the extra field in the form of {"webhook_endpoint": "incomingwebhooks/{webhook.id}?token{webhook.token}"} - :param webhook_endpoint: Discord webhook endpoint in the form of + :param webhook_endpoint: Chime webhook endpoint in the form of "incomingwebhooks/{webhook.id}?token={webhook.token}" - :param message: The message you want to send to your Discord channel + :param message: The message you want to send to your Chime room. (max 4096 characters) """ @@ -50,12 +50,10 @@ def __init__( self, http_conn_id: str | None = None, webhook_endpoint: str | None = None, - message: str = "", *args: Any, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) - self.message = message self.webhook_endpoint = self._get_webhook_endpoint(http_conn_id, webhook_endpoint) def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str | None) -> str: @@ -74,7 +72,9 @@ def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str extra = conn.extra_dejson endpoint = extra.get("webhook_endpoint", "") else: - raise AirflowException() + raise AirflowException( + "Missing one of http_conn_id or webhook_endpoint arguments which are required." + ) # Check to make sure the endpoint matches what Chime expects if not re.match("^incomingwebhooks/[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", endpoint): @@ -85,20 +85,20 @@ def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str return endpoint - def _build_chime_payload(self) -> str: + def _build_chime_payload(self, message: str) -> str: """Builds payload for Chime and ensures messages do not exceed max length allowed.""" payload: dict[str, Any] = {} # We need to make sure that the message does not exceed the max length for Chime - if len(self.message) <= 4096: - payload["Content"] = self.message + if len(message) <= 4096: + payload["Content"] = message else: raise AirflowException("Chime message must be 4096 characters or less.") return json.dumps(payload) - def execute(self) -> None: + def send_message(self, message: str) -> None: """Execute calling the Chime webhook endpoint.""" - chime_payload = self._build_chime_payload() + chime_payload = self._build_chime_payload(message) self.run( endpoint=self.webhook_endpoint, data=chime_payload, headers={"Content-type": "application/json"} ) diff --git a/docs/apache-airflow-providers-amazon/connections/chime.rst b/docs/apache-airflow-providers-amazon/connections/chime.rst new file mode 100644 index 0000000000000..6a83467bb1100 --- /dev/null +++ b/docs/apache-airflow-providers-amazon/connections/chime.rst @@ -0,0 +1,73 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/connection:chime: + +Amazon Chime Connection +========================== + +The Chime connection works with calling Chime webhooks to send messages to a chime room. + +Authenticating to Amazon Chime +--------------------------------- +When a webhook is created in a Chime room a token will be included in the url for authentication. + + +Default Connection IDs +---------------------- + +The default connection ID is ``chime_default``. + +Configuring the Connection +-------------------------- + +Login (optional) + Chime does not require a login for a webhook, this field can be left blank. + +Password (optional) + The token for authentication should be included in extras. No passwords are used for Chime webhooks. + +Host (optional) + Specify the entire url or the base of the url for the service. + +Port (optional) + Specify a port number if applicable. + +Schema (optional) + Specify the service type etc: http/https. + +Extras (optional) + Specify webhook_endpoint here which will start with ``incomingwebhooks/`` +If you are configuring the connection via a URI, ensure that all components of the URI are URL-encoded. + +Examples +-------- + +**Connection** +* **Login: +* **Password: +* **Host: hooks.chime.aws +* **Port: +* ** Schema: https + +* **Extras**: + +.. code-block:: json + +{ +"webhook_endpoint": "incomingwebhooks/abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken" +} From 0e6678350b42a2b83512092eebd8fb795eb5e3fe Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Thu, 15 Jun 2023 15:41:53 -0700 Subject: [PATCH 03/23] Fix chime conn doc formatting --- .../connections/chime.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/apache-airflow-providers-amazon/connections/chime.rst b/docs/apache-airflow-providers-amazon/connections/chime.rst index 6a83467bb1100..a5fb9dc087eb3 100644 --- a/docs/apache-airflow-providers-amazon/connections/chime.rst +++ b/docs/apache-airflow-providers-amazon/connections/chime.rst @@ -58,16 +58,17 @@ Examples -------- **Connection** -* **Login: -* **Password: -* **Host: hooks.chime.aws -* **Port: -* ** Schema: https + +* **Login**: +* **Password**: +* **Host**: hooks.chime.aws +* **Port**: +* **Schema**: https * **Extras**: .. code-block:: json -{ -"webhook_endpoint": "incomingwebhooks/abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken" -} + { + "webhook_endpoint": "incomingwebhooks/abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken" + } From 6cb89ed45354c904ed0d50d634885103b7fa81de Mon Sep 17 00:00:00 2001 From: Beata Kossakowska <109511937+bkossakowska@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:57:39 +0200 Subject: [PATCH 04/23] Fix `BIGQUERY_JOB_DETAILS_LINK_FMT` in `BigQueryConsoleLink` (#31457) Co-authored-by: Beata Kossakowska Co-authored-by: eladkal <45845474+eladkal@users.noreply.github.com> --- .../google/cloud/operators/bigquery.py | 38 +++++++--- .../google/cloud/operators/test_bigquery.py | 69 +++++++++++++------ 2 files changed, 77 insertions(+), 30 deletions(-) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 2fd6aacf5c618..6fb94f21d7496 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -58,7 +58,7 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.utils.context import Context -BIGQUERY_JOB_DETAILS_LINK_FMT = "https://console.cloud.google.com/bigquery?j={job_id}" +BIGQUERY_JOB_DETAILS_LINK_FMT = "https://console.cloud.google.com/bigquery?j={project_id}:{location}:{job_id}" class BigQueryUIColors(enum.Enum): @@ -90,8 +90,17 @@ def get_link( *, ti_key: TaskInstanceKey, ): - job_id = XCom.get_value(key="job_id", ti_key=ti_key) - return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id) if job_id else "" + job_id_params = XCom.get_value(key="job_id_params", ti_key=ti_key) + + return ( + BIGQUERY_JOB_DETAILS_LINK_FMT.format( + job_id=job_id_params["job_id"], + project_id=job_id_params["project_id"], + location=job_id_params["location"], + ) + if job_id_params + else "" + ) @attr.s(auto_attribs=True) @@ -110,13 +119,16 @@ def get_link( *, ti_key: TaskInstanceKey, ): - job_ids = XCom.get_value(key="job_id", ti_key=ti_key) + job_ids_params = XCom.get_value(key="job_id_params", ti_key=ti_key) + job_ids = job_ids_params["job_id"] if not job_ids: return None if len(job_ids) < self.index: return None job_id = job_ids[self.index] - return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id) + return BIGQUERY_JOB_DETAILS_LINK_FMT.format( + job_id=job_id, project_id=job_ids_params["project_id"], location=job_ids_params["location"] + ) class _BigQueryDbHookMixin: @@ -1184,7 +1196,13 @@ def execute(self, context: Context): ] else: raise AirflowException(f"argument 'sql' of type {type(str)} is neither a string nor an iterable") - context["task_instance"].xcom_push(key="job_id", value=job_id) + job_id_params = { + "job_id": job_id, + "project_id": self.hook.project_id, + "location": self.location if self.location else "US", + } + context["task_instance"].xcom_push(key="job_id_params", value=job_id_params) + return job_id def on_kill(self) -> None: super().on_kill() @@ -2727,9 +2745,13 @@ def execute(self, context: Any): persist_kwargs["dataset_id"] = table["datasetId"] persist_kwargs["project_id"] = table["projectId"] BigQueryTableLink.persist(**persist_kwargs) - self.job_id = job.job_id - context["ti"].xcom_push(key="job_id", value=self.job_id) + job_id_params = { + "job_id": job_id, + "project_id": self.project_id or self.hook.project_id, + "location": self.location if self.location else "US", + } + context["ti"].xcom_push(key="job_id_params", value=job_id_params) # Wait for the job to complete if not self.deferrable: job.result(timeout=self.result_timeout, retry=self.result_retry) diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py index 25b341a4c38a5..367ef99cbbe1b 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/tests/providers/google/cloud/operators/test_bigquery.py @@ -83,6 +83,10 @@ } TEST_TABLE = "test-table" GCP_CONN_ID = "google_cloud_default" +TEST_JOB_ID_1 = "test-job-id" +TEST_JOB_ID_2 = "test-123" +TEST_FULL_JOB_ID = f"{TEST_GCP_PROJECT_ID}:{TEST_DATASET_LOCATION}:{TEST_JOB_ID_1}" +TEST_FULL_JOB_ID_2 = f"{TEST_GCP_PROJECT_ID}:{TEST_DATASET_LOCATION}:{TEST_JOB_ID_2}" class TestBigQueryCreateEmptyTableOperator: @@ -672,11 +676,15 @@ def test_bigquery_operator_extra_serialized_field_when_single_query( # Check DeSerialized version of operator link assert isinstance(list(simple_task.operator_extra_links)[0], BigQueryConsoleLink) - - ti.xcom_push("job_id", 12345) + test_job_id_params = { + "job_id": TEST_JOB_ID_1, + "project_id": TEST_GCP_PROJECT_ID, + "location": TEST_DATASET_LOCATION, + } + ti.xcom_push("job_id_params", test_job_id_params) url = simple_task.get_extra_links(ti, BigQueryConsoleLink.name) - assert url == "https://console.cloud.google.com/bigquery?j=12345" + assert url == f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" @pytest.mark.need_serialized_dag def test_bigquery_operator_extra_serialized_field_when_multiple_queries( @@ -711,17 +719,23 @@ def test_bigquery_operator_extra_serialized_field_when_multiple_queries( # Check DeSerialized version of operator link assert isinstance(list(simple_task.operator_extra_links)[0], BigQueryConsoleIndexableLink) - job_id = ["123", "45"] - ti.xcom_push(key="job_id", value=job_id) + test_job_id_params = { + "job_id": [TEST_JOB_ID_1, TEST_JOB_ID_2], + "project_id": TEST_GCP_PROJECT_ID, + "location": TEST_DATASET_LOCATION, + } + ti.xcom_push(key="job_id_params", value=test_job_id_params) assert {"BigQuery Console #1", "BigQuery Console #2"} == simple_task.operator_extra_link_dict.keys() - assert "https://console.cloud.google.com/bigquery?j=123" == simple_task.get_extra_links( - ti, "BigQuery Console #1" + assert ( + f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" + == simple_task.get_extra_links(ti, "BigQuery Console #1") ) - assert "https://console.cloud.google.com/bigquery?j=45" == simple_task.get_extra_links( - ti, "BigQuery Console #2" + assert ( + f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID_2}" + == simple_task.get_extra_links(ti, "BigQuery Console #2") ) @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") @@ -740,7 +754,9 @@ def test_bigquery_operator_extra_link_when_missing_job_id( @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") def test_bigquery_operator_extra_link_when_single_query( - self, mock_hook, create_task_instance_of_operator + self, + mock_hook, + create_task_instance_of_operator, ): ti = create_task_instance_of_operator( BigQueryExecuteQueryOperator, @@ -751,11 +767,15 @@ def test_bigquery_operator_extra_link_when_single_query( ) bigquery_task = ti.task - job_id = "12345" - ti.xcom_push(key="job_id", value=job_id) - - assert f"https://console.cloud.google.com/bigquery?j={job_id}" == bigquery_task.get_extra_links( - ti, BigQueryConsoleLink.name + test_job_id_params = { + "job_id": TEST_JOB_ID_1, + "project_id": TEST_GCP_PROJECT_ID, + "location": TEST_DATASET_LOCATION, + } + ti.xcom_push(key="job_id_params", value=test_job_id_params) + assert ( + f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" + == bigquery_task.get_extra_links(ti, BigQueryConsoleLink.name) ) @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") @@ -771,17 +791,22 @@ def test_bigquery_operator_extra_link_when_multiple_query( ) bigquery_task = ti.task - job_id = ["123", "45"] - ti.xcom_push(key="job_id", value=job_id) - + test_job_id_params = { + "job_id": [TEST_JOB_ID_1, TEST_JOB_ID_2], + "project_id": TEST_GCP_PROJECT_ID, + "location": TEST_DATASET_LOCATION, + } + ti.xcom_push(key="job_id_params", value=test_job_id_params) assert {"BigQuery Console #1", "BigQuery Console #2"} == bigquery_task.operator_extra_link_dict.keys() - assert "https://console.cloud.google.com/bigquery?j=123" == bigquery_task.get_extra_links( - ti, "BigQuery Console #1" + assert ( + f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" + == bigquery_task.get_extra_links(ti, "BigQuery Console #1") ) - assert "https://console.cloud.google.com/bigquery?j=45" == bigquery_task.get_extra_links( - ti, "BigQuery Console #2" + assert ( + f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID_2}" + == bigquery_task.get_extra_links(ti, "BigQuery Console #2") ) From 34fe3b51888a1c0a821afb70cf24a295d199694c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 15 Jun 2023 16:00:06 +0200 Subject: [PATCH 05/23] Revert "Fix `BIGQUERY_JOB_DETAILS_LINK_FMT` in `BigQueryConsoleLink` (#31457)" (#31935) This reverts commit c7072c0490cb80b448622a27eb1056576d6b92a4. --- .../google/cloud/operators/bigquery.py | 38 +++------- .../google/cloud/operators/test_bigquery.py | 69 ++++++------------- 2 files changed, 30 insertions(+), 77 deletions(-) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 6fb94f21d7496..2fd6aacf5c618 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -58,7 +58,7 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.utils.context import Context -BIGQUERY_JOB_DETAILS_LINK_FMT = "https://console.cloud.google.com/bigquery?j={project_id}:{location}:{job_id}" +BIGQUERY_JOB_DETAILS_LINK_FMT = "https://console.cloud.google.com/bigquery?j={job_id}" class BigQueryUIColors(enum.Enum): @@ -90,17 +90,8 @@ def get_link( *, ti_key: TaskInstanceKey, ): - job_id_params = XCom.get_value(key="job_id_params", ti_key=ti_key) - - return ( - BIGQUERY_JOB_DETAILS_LINK_FMT.format( - job_id=job_id_params["job_id"], - project_id=job_id_params["project_id"], - location=job_id_params["location"], - ) - if job_id_params - else "" - ) + job_id = XCom.get_value(key="job_id", ti_key=ti_key) + return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id) if job_id else "" @attr.s(auto_attribs=True) @@ -119,16 +110,13 @@ def get_link( *, ti_key: TaskInstanceKey, ): - job_ids_params = XCom.get_value(key="job_id_params", ti_key=ti_key) - job_ids = job_ids_params["job_id"] + job_ids = XCom.get_value(key="job_id", ti_key=ti_key) if not job_ids: return None if len(job_ids) < self.index: return None job_id = job_ids[self.index] - return BIGQUERY_JOB_DETAILS_LINK_FMT.format( - job_id=job_id, project_id=job_ids_params["project_id"], location=job_ids_params["location"] - ) + return BIGQUERY_JOB_DETAILS_LINK_FMT.format(job_id=job_id) class _BigQueryDbHookMixin: @@ -1196,13 +1184,7 @@ def execute(self, context: Context): ] else: raise AirflowException(f"argument 'sql' of type {type(str)} is neither a string nor an iterable") - job_id_params = { - "job_id": job_id, - "project_id": self.hook.project_id, - "location": self.location if self.location else "US", - } - context["task_instance"].xcom_push(key="job_id_params", value=job_id_params) - return job_id + context["task_instance"].xcom_push(key="job_id", value=job_id) def on_kill(self) -> None: super().on_kill() @@ -2745,13 +2727,9 @@ def execute(self, context: Any): persist_kwargs["dataset_id"] = table["datasetId"] persist_kwargs["project_id"] = table["projectId"] BigQueryTableLink.persist(**persist_kwargs) + self.job_id = job.job_id - job_id_params = { - "job_id": job_id, - "project_id": self.project_id or self.hook.project_id, - "location": self.location if self.location else "US", - } - context["ti"].xcom_push(key="job_id_params", value=job_id_params) + context["ti"].xcom_push(key="job_id", value=self.job_id) # Wait for the job to complete if not self.deferrable: job.result(timeout=self.result_timeout, retry=self.result_retry) diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py index 367ef99cbbe1b..25b341a4c38a5 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/tests/providers/google/cloud/operators/test_bigquery.py @@ -83,10 +83,6 @@ } TEST_TABLE = "test-table" GCP_CONN_ID = "google_cloud_default" -TEST_JOB_ID_1 = "test-job-id" -TEST_JOB_ID_2 = "test-123" -TEST_FULL_JOB_ID = f"{TEST_GCP_PROJECT_ID}:{TEST_DATASET_LOCATION}:{TEST_JOB_ID_1}" -TEST_FULL_JOB_ID_2 = f"{TEST_GCP_PROJECT_ID}:{TEST_DATASET_LOCATION}:{TEST_JOB_ID_2}" class TestBigQueryCreateEmptyTableOperator: @@ -676,15 +672,11 @@ def test_bigquery_operator_extra_serialized_field_when_single_query( # Check DeSerialized version of operator link assert isinstance(list(simple_task.operator_extra_links)[0], BigQueryConsoleLink) - test_job_id_params = { - "job_id": TEST_JOB_ID_1, - "project_id": TEST_GCP_PROJECT_ID, - "location": TEST_DATASET_LOCATION, - } - ti.xcom_push("job_id_params", test_job_id_params) + + ti.xcom_push("job_id", 12345) url = simple_task.get_extra_links(ti, BigQueryConsoleLink.name) - assert url == f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" + assert url == "https://console.cloud.google.com/bigquery?j=12345" @pytest.mark.need_serialized_dag def test_bigquery_operator_extra_serialized_field_when_multiple_queries( @@ -719,23 +711,17 @@ def test_bigquery_operator_extra_serialized_field_when_multiple_queries( # Check DeSerialized version of operator link assert isinstance(list(simple_task.operator_extra_links)[0], BigQueryConsoleIndexableLink) - test_job_id_params = { - "job_id": [TEST_JOB_ID_1, TEST_JOB_ID_2], - "project_id": TEST_GCP_PROJECT_ID, - "location": TEST_DATASET_LOCATION, - } - ti.xcom_push(key="job_id_params", value=test_job_id_params) + job_id = ["123", "45"] + ti.xcom_push(key="job_id", value=job_id) assert {"BigQuery Console #1", "BigQuery Console #2"} == simple_task.operator_extra_link_dict.keys() - assert ( - f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" - == simple_task.get_extra_links(ti, "BigQuery Console #1") + assert "https://console.cloud.google.com/bigquery?j=123" == simple_task.get_extra_links( + ti, "BigQuery Console #1" ) - assert ( - f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID_2}" - == simple_task.get_extra_links(ti, "BigQuery Console #2") + assert "https://console.cloud.google.com/bigquery?j=45" == simple_task.get_extra_links( + ti, "BigQuery Console #2" ) @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") @@ -754,9 +740,7 @@ def test_bigquery_operator_extra_link_when_missing_job_id( @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") def test_bigquery_operator_extra_link_when_single_query( - self, - mock_hook, - create_task_instance_of_operator, + self, mock_hook, create_task_instance_of_operator ): ti = create_task_instance_of_operator( BigQueryExecuteQueryOperator, @@ -767,15 +751,11 @@ def test_bigquery_operator_extra_link_when_single_query( ) bigquery_task = ti.task - test_job_id_params = { - "job_id": TEST_JOB_ID_1, - "project_id": TEST_GCP_PROJECT_ID, - "location": TEST_DATASET_LOCATION, - } - ti.xcom_push(key="job_id_params", value=test_job_id_params) - assert ( - f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" - == bigquery_task.get_extra_links(ti, BigQueryConsoleLink.name) + job_id = "12345" + ti.xcom_push(key="job_id", value=job_id) + + assert f"https://console.cloud.google.com/bigquery?j={job_id}" == bigquery_task.get_extra_links( + ti, BigQueryConsoleLink.name ) @mock.patch("airflow.providers.google.cloud.operators.bigquery.BigQueryHook") @@ -791,22 +771,17 @@ def test_bigquery_operator_extra_link_when_multiple_query( ) bigquery_task = ti.task - test_job_id_params = { - "job_id": [TEST_JOB_ID_1, TEST_JOB_ID_2], - "project_id": TEST_GCP_PROJECT_ID, - "location": TEST_DATASET_LOCATION, - } - ti.xcom_push(key="job_id_params", value=test_job_id_params) + job_id = ["123", "45"] + ti.xcom_push(key="job_id", value=job_id) + assert {"BigQuery Console #1", "BigQuery Console #2"} == bigquery_task.operator_extra_link_dict.keys() - assert ( - f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID}" - == bigquery_task.get_extra_links(ti, "BigQuery Console #1") + assert "https://console.cloud.google.com/bigquery?j=123" == bigquery_task.get_extra_links( + ti, "BigQuery Console #1" ) - assert ( - f"https://console.cloud.google.com/bigquery?j={TEST_FULL_JOB_ID_2}" - == bigquery_task.get_extra_links(ti, "BigQuery Console #2") + assert "https://console.cloud.google.com/bigquery?j=45" == bigquery_task.get_extra_links( + ti, "BigQuery Console #2" ) From 7ac8ebebf341b911d1363bdda0f2ea27b8fee14b Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Thu, 15 Jun 2023 19:56:31 -0700 Subject: [PATCH 06/23] Fix Chime docs, add missing deps --- airflow/providers/amazon/aws/hooks/chime.py | 28 ++++++++++++------- airflow/providers/amazon/provider.yaml | 6 ++++ .../connections/chime.rst | 2 +- .../apache-airflow-providers-amazon/index.rst | 1 + generated/provider_dependencies.json | 1 + 5 files changed, 27 insertions(+), 11 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index aaba80f2399ab..b1e2c54ca1cb2 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -30,15 +30,13 @@ class ChimeWebhookHook(HttpHook): """Interact with Chime Web Hooks to create notifications. - .. warning:: This hook is only designed to work with Web Hooks and not Chatbots. + .. warning:: This hook is only designed to work with web hooks and not chat bots. :param http_conn_id: Http connection ID with host as "https://hooks.chime.aws" and default webhook endpoint in the extra field in the form of - {"webhook_endpoint": "incomingwebhooks/{webhook.id}?token{webhook.token}"} + ``{"webhook_endpoint": "incomingwebhooks/{webhook.id}?token{webhook.token}"}`` :param webhook_endpoint: Chime webhook endpoint in the form of - "incomingwebhooks/{webhook.id}?token={webhook.token}" - :param message: The message you want to send to your Chime room. - (max 4096 characters) + ``"incomingwebhooks/{webhook.id}?token={webhook.token}"`` """ conn_name_attr = "http_conn_id" @@ -61,9 +59,9 @@ def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str Given a Chime http_conn_id return the default webhook endpoint or override if webhook_endpoint is manually provided. - :param http_conn_id: The provided connection ID - :param webhook_endpoint: The manually provided webhook endpoint - :return: Webhook Endpoint(str) to use with Chime + :param http_conn_id: The provided connection ID. + :param webhook_endpoint: The manually provided endpoint for the chime webhook. + :return: Endpoint(str) for chime webhook. """ if webhook_endpoint: endpoint = webhook_endpoint @@ -86,7 +84,12 @@ def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str return endpoint def _build_chime_payload(self, message: str) -> str: - """Builds payload for Chime and ensures messages do not exceed max length allowed.""" + """ + Builds payload for Chime and ensures messages do not exceed max length allowed. + + :param message: The message you want to send to your Chime room. + (max 4096 characters) + """ payload: dict[str, Any] = {} # We need to make sure that the message does not exceed the max length for Chime if len(message) <= 4096: @@ -97,7 +100,12 @@ def _build_chime_payload(self, message: str) -> str: return json.dumps(payload) def send_message(self, message: str) -> None: - """Execute calling the Chime webhook endpoint.""" + """Execute calling the Chime webhook endpoint. + + :param message: The message you want to send to your Chime room. + (max 4096 characters) + + """ chime_payload = self._build_chime_payload(message) self.run( endpoint=self.webhook_endpoint, data=chime_payload, headers={"Content-type": "application/json"} diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 1c138a495cc1f..d07375aed7efc 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -60,6 +60,7 @@ versions: dependencies: - apache-airflow>=2.4.0 - apache-airflow-providers-common-sql>=1.3.1 + - apache-airflow-providers-http - boto3>=1.24.0 - asgiref # watchtower 3 has been released end Jan and introduced breaking change across the board that might @@ -83,6 +84,11 @@ integrations: how-to-guide: - /docs/apache-airflow-providers-amazon/operators/athena.rst tags: [aws] + - integration-name: Amazon Chime + external-doc-url: https://aws.amazon.com/chime/ + how-to-guide: + - /docs/apache-airflow-providers-amazon/connections/chime.rst + tags: [aws] - integration-name: Amazon CloudFormation external-doc-url: https://aws.amazon.com/cloudformation/ logo: /integration-logos/aws/AWS-CloudFormation_light-bg@4x.png diff --git a/docs/apache-airflow-providers-amazon/connections/chime.rst b/docs/apache-airflow-providers-amazon/connections/chime.rst index a5fb9dc087eb3..aa67632b1b24e 100644 --- a/docs/apache-airflow-providers-amazon/connections/chime.rst +++ b/docs/apache-airflow-providers-amazon/connections/chime.rst @@ -52,7 +52,7 @@ Schema (optional) Extras (optional) Specify webhook_endpoint here which will start with ``incomingwebhooks/`` -If you are configuring the connection via a URI, ensure that all components of the URI are URL-encoded. + Examples -------- diff --git a/docs/apache-airflow-providers-amazon/index.rst b/docs/apache-airflow-providers-amazon/index.rst index e063c447d4bec..ac598fb393031 100644 --- a/docs/apache-airflow-providers-amazon/index.rst +++ b/docs/apache-airflow-providers-amazon/index.rst @@ -133,6 +133,7 @@ Dependent package `apache-airflow-providers-exasol `_ ``exasol`` `apache-airflow-providers-ftp `_ ``ftp`` `apache-airflow-providers-google `_ ``google`` +`apache-airflow-providers-http `_ ``http`` `apache-airflow-providers-imap `_ ``imap`` `apache-airflow-providers-mongo `_ ``mongo`` `apache-airflow-providers-salesforce `_ ``salesforce`` diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 50d69b309ff65..871075d9300e8 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -20,6 +20,7 @@ "amazon": { "deps": [ "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow-providers-http", "apache-airflow>=2.4.0", "asgiref", "asgiref", From 71fcf702c9b550f35b018a032f1c694fe6e02c29 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Fri, 16 Jun 2023 10:05:36 -0700 Subject: [PATCH 07/23] Handle missing chime extras --- airflow/providers/amazon/aws/hooks/chime.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index b1e2c54ca1cb2..cf102a1981f21 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -69,6 +69,8 @@ def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str conn = self.get_connection(http_conn_id) extra = conn.extra_dejson endpoint = extra.get("webhook_endpoint", "") + if endpoint == "": + raise AirflowException("webhook_endpoint missing from extras and is required.") else: raise AirflowException( "Missing one of http_conn_id or webhook_endpoint arguments which are required." @@ -92,11 +94,10 @@ def _build_chime_payload(self, message: str) -> str: """ payload: dict[str, Any] = {} # We need to make sure that the message does not exceed the max length for Chime - if len(message) <= 4096: - payload["Content"] = message - else: + if len(message) > 4096: raise AirflowException("Chime message must be 4096 characters or less.") + payload["Content"] = message return json.dumps(payload) def send_message(self, message: str) -> None: From e19b725f4849e755ff896b4ea4736b2f11c9ebf1 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Fri, 16 Jun 2023 10:45:34 -0700 Subject: [PATCH 08/23] Add test, change endpoint to check none --- airflow/providers/amazon/aws/hooks/chime.py | 4 ++-- tests/providers/amazon/aws/hooks/test_chime.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index cf102a1981f21..b8a535b743fb3 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -68,8 +68,8 @@ def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str elif http_conn_id: conn = self.get_connection(http_conn_id) extra = conn.extra_dejson - endpoint = extra.get("webhook_endpoint", "") - if endpoint == "": + endpoint = extra.get("webhook_endpoint") + if endpoint is None: raise AirflowException("webhook_endpoint missing from extras and is required.") else: raise AirflowException( diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/tests/providers/amazon/aws/hooks/test_chime.py index fafe9c5e69d92..f4d6e5a476d7f 100644 --- a/tests/providers/amazon/aws/hooks/test_chime.py +++ b/tests/providers/amazon/aws/hooks/test_chime.py @@ -50,6 +50,13 @@ def setup_method(self): extra='{"webhook_endpoint": "incomingwebhooks/abcd-1134?token=somechimetoken_111"}', ) ) + db.merge_conn( + Connection( + conn_id="chime-webhook-missing-extra", + conn_type="chime", + host="https://hooks.chime.aws" + ) + ) def test_get_webhook_endpoint_manual_token(self): # Given @@ -71,6 +78,14 @@ def test_get_webhook_endpoint_invalid_url(self): with pytest.raises(AirflowException, match=expected_message): ChimeWebhookHook(webhook_endpoint=provided_endpoint) + def test_get_webhook_endpoint_conn_id_missing_extras(self): + # Given + conn_id = "chime-webhook-missing-extra" + hook = ChimeWebhookHook(http_conn_id="chime-webhook-missing-extra") + expected_message = "webhook_endpoint missing from extras and is required." + with pytest.raises(AirflowException, match=expected_message): + hook._get_webhook_endpoint(conn_id) + def test_get_webhook_endpoint_conn_id(self): # Given conn_id = "default-chime-webhook" From 2c7e1db43dd0f705b56e3083ad3a592b35af7d64 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Mon, 19 Jun 2023 08:38:58 -0700 Subject: [PATCH 09/23] Use custom fields for Chime conn --- airflow/providers/amazon/aws/hooks/chime.py | 56 ++++++++++--------- .../providers/amazon/aws/hooks/test_chime.py | 38 ++----------- 2 files changed, 36 insertions(+), 58 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index b8a535b743fb3..cc99cd8fb35ad 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -32,11 +32,9 @@ class ChimeWebhookHook(HttpHook): .. warning:: This hook is only designed to work with web hooks and not chat bots. - :param http_conn_id: Http connection ID with host as "https://hooks.chime.aws" and - default webhook endpoint in the extra field in the form of - ``{"webhook_endpoint": "incomingwebhooks/{webhook.id}?token{webhook.token}"}`` - :param webhook_endpoint: Chime webhook endpoint in the form of - ``"incomingwebhooks/{webhook.id}?token={webhook.token}"`` + :param chime_conn_id: Chime connection ID with Endpoint as "https://hooks.chime.aws" and + the webhook token in the form of ```{webhook.id}?token{webhook.token}``` + """ conn_name_attr = "http_conn_id" @@ -46,41 +44,31 @@ class ChimeWebhookHook(HttpHook): def __init__( self, - http_conn_id: str | None = None, - webhook_endpoint: str | None = None, + chime_conn_id: str, *args: Any, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) - self.webhook_endpoint = self._get_webhook_endpoint(http_conn_id, webhook_endpoint) + self.webhook_endpoint = self._get_webhook_endpoint(chime_conn_id) - def _get_webhook_endpoint(self, http_conn_id: str | None, webhook_endpoint: str | None) -> str: + def _get_webhook_endpoint(self, conn_id: str | None) -> str: """ Given a Chime http_conn_id return the default webhook endpoint or override if webhook_endpoint is manually provided. - :param http_conn_id: The provided connection ID. - :param webhook_endpoint: The manually provided endpoint for the chime webhook. + :param conn_id: The provided connection ID. :return: Endpoint(str) for chime webhook. """ - if webhook_endpoint: - endpoint = webhook_endpoint - elif http_conn_id: - conn = self.get_connection(http_conn_id) - extra = conn.extra_dejson - endpoint = extra.get("webhook_endpoint") - if endpoint is None: - raise AirflowException("webhook_endpoint missing from extras and is required.") - else: - raise AirflowException( - "Missing one of http_conn_id or webhook_endpoint arguments which are required." - ) + conn = self.get_connection(conn_id) + token = conn.get_password() + url = conn.schema + "://" + conn.host + endpoint = url + token # Check to make sure the endpoint matches what Chime expects - if not re.match("^incomingwebhooks/[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", endpoint): + if not re.match("^incomingwebhooks/[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token): raise AirflowException( - "Expected Chime webhook endpoint in the form of " - '"incomingwebhooks/{webhook.id}?token={webhook.token}".' + "Expected Chime webhook token in the form of " + '"{webhook.id}?token={webhook.token}".' ) return endpoint @@ -111,3 +99,19 @@ def send_message(self, message: str) -> None: self.run( endpoint=self.webhook_endpoint, data=chime_payload, headers={"Content-type": "application/json"} ) + + @classmethod + def get_ui_field_behaviour(cls) -> dict[str, Any]: + """Returns custom field behaviour to only get what is needed for Chime webhooks to function.""" + return { + "hidden_fields": ["login", "port", "extra"], + "relabeling": { + "host": "Chime Webhook Endpoint", + "password": "Webhook Token", + }, + "placeholders": { + "schema": "https", + "host": "hooks.chime.aws/incomingwebhook/", + "password": "T00000000?token=XXXXXXXXXXXXXXXXXXXXXXXX" + }, + } diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/tests/providers/amazon/aws/hooks/test_chime.py index f4d6e5a476d7f..c2ee1ce25dbf3 100644 --- a/tests/providers/amazon/aws/hooks/test_chime.py +++ b/tests/providers/amazon/aws/hooks/test_chime.py @@ -30,7 +30,7 @@ class TestChimeWebhookHook: _config = { - "http_conn_id": "default-chime-webhook", + "chime_conn_id": "default-chime-webhook", "webhook_endpoint": "incomingwebhooks/abcd-1134?token=somechimetoken_111", "message": "your message here", } @@ -46,28 +46,10 @@ def setup_method(self): Connection( conn_id="default-chime-webhook", conn_type="chime", - host="https://hooks.chime.aws", - extra='{"webhook_endpoint": "incomingwebhooks/abcd-1134?token=somechimetoken_111"}', + host="https://hooks.chime.aws/incomingwebhooks/", + password="incomingwebhooks/abcd-1134?token=somechimetoken_111", ) ) - db.merge_conn( - Connection( - conn_id="chime-webhook-missing-extra", - conn_type="chime", - host="https://hooks.chime.aws" - ) - ) - - def test_get_webhook_endpoint_manual_token(self): - # Given - provided_endpoint = "incomingwebhooks/abcd-1134?token=somechimetoken_111" - hook = ChimeWebhookHook(webhook_endpoint=provided_endpoint) - - # When - webhook_endpoint = hook._get_webhook_endpoint(None, provided_endpoint) - - # Then - assert webhook_endpoint == provided_endpoint def test_get_webhook_endpoint_invalid_url(self): # Given @@ -78,22 +60,14 @@ def test_get_webhook_endpoint_invalid_url(self): with pytest.raises(AirflowException, match=expected_message): ChimeWebhookHook(webhook_endpoint=provided_endpoint) - def test_get_webhook_endpoint_conn_id_missing_extras(self): - # Given - conn_id = "chime-webhook-missing-extra" - hook = ChimeWebhookHook(http_conn_id="chime-webhook-missing-extra") - expected_message = "webhook_endpoint missing from extras and is required." - with pytest.raises(AirflowException, match=expected_message): - hook._get_webhook_endpoint(conn_id) - def test_get_webhook_endpoint_conn_id(self): # Given conn_id = "default-chime-webhook" - hook = ChimeWebhookHook(http_conn_id=conn_id) - expected_webhook_endpoint = "incomingwebhooks/abcd-1134?token=somechimetoken_111" + hook = ChimeWebhookHook(chime_conn_id=conn_id) + expected_webhook_endpoint = "https://hooks.chime.aws/incomingwebhooks/abcd-1134?token=somechimetoken_111" # When - webhook_endpoint = hook._get_webhook_endpoint(conn_id, None) + webhook_endpoint = hook._get_webhook_endpoint(conn_id) # Then assert webhook_endpoint == expected_webhook_endpoint From e1a0209b14c2ccc546d1a3deb590c2de6f5ade0d Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Mon, 19 Jun 2023 08:42:36 -0700 Subject: [PATCH 10/23] Adjust method types --- airflow/providers/amazon/aws/hooks/chime.py | 8 +++----- tests/providers/amazon/aws/hooks/test_chime.py | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index cc99cd8fb35ad..002a17c831b91 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -51,7 +51,7 @@ def __init__( super().__init__(*args, **kwargs) self.webhook_endpoint = self._get_webhook_endpoint(chime_conn_id) - def _get_webhook_endpoint(self, conn_id: str | None) -> str: + def _get_webhook_endpoint(self, conn_id: str) -> str: """ Given a Chime http_conn_id return the default webhook endpoint or override if webhook_endpoint is manually provided. @@ -59,7 +59,6 @@ def _get_webhook_endpoint(self, conn_id: str | None) -> str: :param conn_id: The provided connection ID. :return: Endpoint(str) for chime webhook. """ - conn = self.get_connection(conn_id) token = conn.get_password() url = conn.schema + "://" + conn.host @@ -67,8 +66,7 @@ def _get_webhook_endpoint(self, conn_id: str | None) -> str: # Check to make sure the endpoint matches what Chime expects if not re.match("^incomingwebhooks/[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token): raise AirflowException( - "Expected Chime webhook token in the form of " - '"{webhook.id}?token={webhook.token}".' + "Expected Chime webhook token in the form of " '"{webhook.id}?token={webhook.token}".' ) return endpoint @@ -112,6 +110,6 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]: "placeholders": { "schema": "https", "host": "hooks.chime.aws/incomingwebhook/", - "password": "T00000000?token=XXXXXXXXXXXXXXXXXXXXXXXX" + "password": "T00000000?token=XXXXXXXXXXXXXXXXXXXXXXXX", }, } diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/tests/providers/amazon/aws/hooks/test_chime.py index c2ee1ce25dbf3..7b16fa48d0d15 100644 --- a/tests/providers/amazon/aws/hooks/test_chime.py +++ b/tests/providers/amazon/aws/hooks/test_chime.py @@ -64,7 +64,9 @@ def test_get_webhook_endpoint_conn_id(self): # Given conn_id = "default-chime-webhook" hook = ChimeWebhookHook(chime_conn_id=conn_id) - expected_webhook_endpoint = "https://hooks.chime.aws/incomingwebhooks/abcd-1134?token=somechimetoken_111" + expected_webhook_endpoint = ( + "https://hooks.chime.aws/incomingwebhooks/abcd-1134?token=somechimetoken_111" + ) # When webhook_endpoint = hook._get_webhook_endpoint(conn_id) From 50f334a96f6505198c64793b2b7c4c87dc91f405 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Tue, 20 Jun 2023 07:39:54 -0700 Subject: [PATCH 11/23] Fix implicit str concat --- airflow/providers/amazon/aws/hooks/chime.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index 002a17c831b91..ebf55e5a53055 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -37,7 +37,7 @@ class ChimeWebhookHook(HttpHook): """ - conn_name_attr = "http_conn_id" + conn_name_attr = "chime_conn_id" default_conn_name = "chime_default" conn_type = "chime" hook_name = "Chime Web Hook" @@ -61,12 +61,14 @@ def _get_webhook_endpoint(self, conn_id: str) -> str: """ conn = self.get_connection(conn_id) token = conn.get_password() + if token is None: + raise AirflowException("Webhook token field is missing and is required.") url = conn.schema + "://" + conn.host endpoint = url + token # Check to make sure the endpoint matches what Chime expects if not re.match("^incomingwebhooks/[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token): raise AirflowException( - "Expected Chime webhook token in the form of " '"{webhook.id}?token={webhook.token}".' + "Expected Chime webhook token in the form of '{webhook.id}?token={webhook.token}'." ) return endpoint From d9d79f9c57d9b2536991adcfd98f2fcbd595ba14 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Tue, 20 Jun 2023 08:45:32 -0700 Subject: [PATCH 12/23] Fix Chime Conn docs --- .../connections/chime.rst | 35 ++++++------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/docs/apache-airflow-providers-amazon/connections/chime.rst b/docs/apache-airflow-providers-amazon/connections/chime.rst index aa67632b1b24e..b71d33bf9f52a 100644 --- a/docs/apache-airflow-providers-amazon/connections/chime.rst +++ b/docs/apache-airflow-providers-amazon/connections/chime.rst @@ -34,24 +34,15 @@ The default connection ID is ``chime_default``. Configuring the Connection -------------------------- - -Login (optional) - Chime does not require a login for a webhook, this field can be left blank. - -Password (optional) - The token for authentication should be included in extras. No passwords are used for Chime webhooks. - -Host (optional) +Chime Webhook Endpoint: Specify the entire url or the base of the url for the service. -Port (optional) - Specify a port number if applicable. -Schema (optional) - Specify the service type etc: http/https. +Chime Webhook token: + The token for authentication including the webhook ID. -Extras (optional) - Specify webhook_endpoint here which will start with ``incomingwebhooks/`` +Schema: + Whether or not the endpoint should be http or https Examples @@ -59,16 +50,10 @@ Examples **Connection** -* **Login**: -* **Password**: -* **Host**: hooks.chime.aws -* **Port**: -* **Schema**: https - -* **Extras**: +* **Chime Webhook Endpoint**: hooks.chime.aws +* **Chime Webhook Token**: -.. code-block:: json +.. code-block:: text - { - "webhook_endpoint": "incomingwebhooks/abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken" - } + abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken +* **Schema**: https From b0786de7873435ff9a7f55115efb0d830b35959b Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Thu, 22 Jun 2023 21:23:12 -0700 Subject: [PATCH 13/23] Fix Chime Hook tests --- airflow/providers/amazon/aws/hooks/chime.py | 5 ++- .../providers/amazon/aws/hooks/test_chime.py | 35 ++++++++++++------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index ebf55e5a53055..df61272335adf 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -53,8 +53,7 @@ def __init__( def _get_webhook_endpoint(self, conn_id: str) -> str: """ - Given a Chime http_conn_id return the default webhook endpoint or override if - webhook_endpoint is manually provided. + Given a Chime conn_id return the default webhook endpoint :param conn_id: The provided connection ID. :return: Endpoint(str) for chime webhook. @@ -66,7 +65,7 @@ def _get_webhook_endpoint(self, conn_id: str) -> str: url = conn.schema + "://" + conn.host endpoint = url + token # Check to make sure the endpoint matches what Chime expects - if not re.match("^incomingwebhooks/[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token): + if not re.match(r"^[a-zA-Z0-9_-]+\?token=[a-zA-Z0-9_-]+$", token): raise AirflowException( "Expected Chime webhook token in the form of '{webhook.id}?token={webhook.token}'." ) diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/tests/providers/amazon/aws/hooks/test_chime.py index 7b16fa48d0d15..17389ce15d24d 100644 --- a/tests/providers/amazon/aws/hooks/test_chime.py +++ b/tests/providers/amazon/aws/hooks/test_chime.py @@ -31,7 +31,7 @@ class TestChimeWebhookHook: _config = { "chime_conn_id": "default-chime-webhook", - "webhook_endpoint": "incomingwebhooks/abcd-1134?token=somechimetoken_111", + "webhook_endpoint": "incomingwebhooks/abcd-1134-ZeDA?token=somechimetoken-111", "message": "your message here", } @@ -46,8 +46,18 @@ def setup_method(self): Connection( conn_id="default-chime-webhook", conn_type="chime", - host="https://hooks.chime.aws/incomingwebhooks/", - password="incomingwebhooks/abcd-1134?token=somechimetoken_111", + host="hooks.chime.aws/incomingwebhooks/", + password="abcd-1134-ZeDA?token=somechimetoken111", + schema="https" + ) + ) + db.merge_conn( + Connection( + conn_id="chime-bad-url", + conn_type="chime", + host="https://hooks.chime.aws/", + password="somebadurl", + schema="https" ) ) @@ -56,16 +66,16 @@ def test_get_webhook_endpoint_invalid_url(self): provided_endpoint = "https://hooks.chime.aws/some-invalid-webhook-url" # When/Then - expected_message = "Expected Chime webhook endpoint in the form of" + expected_message = r"Expected Chime webhook token in the form" with pytest.raises(AirflowException, match=expected_message): - ChimeWebhookHook(webhook_endpoint=provided_endpoint) + ChimeWebhookHook(chime_conn_id="chime-bad-url") def test_get_webhook_endpoint_conn_id(self): # Given conn_id = "default-chime-webhook" hook = ChimeWebhookHook(chime_conn_id=conn_id) expected_webhook_endpoint = ( - "https://hooks.chime.aws/incomingwebhooks/abcd-1134?token=somechimetoken_111" + "https://hooks.chime.aws/incomingwebhooks/abcd-1134-ZeDA?token=somechimetoken111" ) # When @@ -76,11 +86,10 @@ def test_get_webhook_endpoint_conn_id(self): def test_build_chime_payload(self): # Given - hook = ChimeWebhookHook(**self._config) - + hook = ChimeWebhookHook(self._config["chime_conn_id"]) + message = self._config["message"] # When - payload = hook._build_chime_payload() - + payload = hook._build_chime_payload(message) # Then assert self.expected_payload == payload @@ -88,10 +97,10 @@ def test_build_chime_payload_message_length(self): # Given config = self._config.copy() # create message over the character limit - config["message"] = "c" * 4097 - hook = ChimeWebhookHook(**config) + message = "c" * 4097 + hook = ChimeWebhookHook(self._config["chime_conn_id"]) # When/Then expected_message = "Chime message must be 4096 characters or less." with pytest.raises(AirflowException, match=expected_message): - hook._build_chime_payload() + hook._build_chime_payload(message) From 0cdd5bc7fbce0256f63a7e96e6276071e0f21fd9 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Thu, 22 Jun 2023 21:24:46 -0700 Subject: [PATCH 14/23] Fix Chime Hook tests --- airflow/providers/amazon/aws/hooks/chime.py | 2 +- tests/providers/amazon/aws/hooks/test_chime.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/airflow/providers/amazon/aws/hooks/chime.py b/airflow/providers/amazon/aws/hooks/chime.py index df61272335adf..e2ac1a35a0d9b 100644 --- a/airflow/providers/amazon/aws/hooks/chime.py +++ b/airflow/providers/amazon/aws/hooks/chime.py @@ -53,7 +53,7 @@ def __init__( def _get_webhook_endpoint(self, conn_id: str) -> str: """ - Given a Chime conn_id return the default webhook endpoint + Given a Chime conn_id return the default webhook endpoint. :param conn_id: The provided connection ID. :return: Endpoint(str) for chime webhook. diff --git a/tests/providers/amazon/aws/hooks/test_chime.py b/tests/providers/amazon/aws/hooks/test_chime.py index 17389ce15d24d..d5130964f4d27 100644 --- a/tests/providers/amazon/aws/hooks/test_chime.py +++ b/tests/providers/amazon/aws/hooks/test_chime.py @@ -48,7 +48,7 @@ def setup_method(self): conn_type="chime", host="hooks.chime.aws/incomingwebhooks/", password="abcd-1134-ZeDA?token=somechimetoken111", - schema="https" + schema="https", ) ) db.merge_conn( @@ -57,13 +57,12 @@ def setup_method(self): conn_type="chime", host="https://hooks.chime.aws/", password="somebadurl", - schema="https" + schema="https", ) ) def test_get_webhook_endpoint_invalid_url(self): # Given - provided_endpoint = "https://hooks.chime.aws/some-invalid-webhook-url" # When/Then expected_message = r"Expected Chime webhook token in the form" @@ -95,7 +94,7 @@ def test_build_chime_payload(self): def test_build_chime_payload_message_length(self): # Given - config = self._config.copy() + self._config.copy() # create message over the character limit message = "c" * 4097 hook = ChimeWebhookHook(self._config["chime_conn_id"]) From f6bd7623a3a6d2d88bec6b2821a265c3f67aa879 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Thu, 22 Jun 2023 22:40:21 -0700 Subject: [PATCH 15/23] Fix Chime Docs --- airflow/providers/amazon/provider.yaml | 5 +++-- docs/apache-airflow-providers-amazon/connections/chime.rst | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 54b4e1f14b171..2cc9c194ed7c2 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -630,12 +630,13 @@ extra-links: connection-types: - hook-class-name: airflow.providers.amazon.aws.hooks.base_aws.AwsGenericHook connection-type: aws + - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook + connection-type: chime - hook-class-name: airflow.providers.amazon.aws.hooks.emr.EmrHook connection-type: emr - hook-class-name: airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook connection-type: redshift - - hook-class-name: airflow.providers.amazon.aws.hooks.chime.ChimeWebhookHook - connection-type: chime + secrets-backends: - airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend diff --git a/docs/apache-airflow-providers-amazon/connections/chime.rst b/docs/apache-airflow-providers-amazon/connections/chime.rst index b71d33bf9f52a..94a356f81d805 100644 --- a/docs/apache-airflow-providers-amazon/connections/chime.rst +++ b/docs/apache-airflow-providers-amazon/connections/chime.rst @@ -56,4 +56,5 @@ Examples .. code-block:: text abceasd-3423-a1237-ffff-000cccccccc?token=somechimetoken + * **Schema**: https From 25da11c6c534a08d8f1e4b83f4badfbadc5e5e33 Mon Sep 17 00:00:00 2001 From: Cary Hawkins Date: Fri, 23 Jun 2023 09:29:41 -0700 Subject: [PATCH 16/23] Add chime logo --- airflow/providers/amazon/provider.yaml | 3 +-- .../aws/Amazon-Chime-light-bg.png | Bin 0 -> 9969 bytes 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 docs/integration-logos/aws/Amazon-Chime-light-bg.png diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 2cc9c194ed7c2..cb7961eb6499e 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -87,8 +87,7 @@ integrations: tags: [aws] - integration-name: Amazon Chime external-doc-url: https://aws.amazon.com/chime/ - how-to-guide: - - /docs/apache-airflow-providers-amazon/connections/chime.rst + logo: /integration-logos/aws/Amazon-Chime-light-bg.png tags: [aws] - integration-name: Amazon CloudFormation external-doc-url: https://aws.amazon.com/cloudformation/ diff --git a/docs/integration-logos/aws/Amazon-Chime-light-bg.png b/docs/integration-logos/aws/Amazon-Chime-light-bg.png new file mode 100644 index 0000000000000000000000000000000000000000..bd412e98151474810392e9e3d7c195ecfe58b261 GIT binary patch literal 9969 zcmVo001BWNkl_XVw^4uWB7);=%y47W-hW)jHYNM5u+(X2%m`CzT?VDS3c~3A4SRsqyu?}k z|5r%~D;;`2JF;P;BQeIRlOR|NeX!bILojO6Yp>Y$W8~n_H{Y_-haYvtcdR%~QJJ-{ zt1koWE?V56J6Wx3df9bjcs-;*$-Vz_#doYYO<59<0fk+=#Y)VcjWcP=s|@Vwh8>*0 zfR#P-tQ)Rl#%IcsMT%zi!D?^Kj>*bm$^UuNo|7m`+CgL&F%@ zO*ODkF%=|7Hf)H>yi&jTTz8+pN=sScWq+5tzmcEv8MD~_88fB)9F2pK8?IUNyTmS7 z6Er|o7wXW8l?2$KuCE&yJ&~AIYX?YUmP!;wW`4lVhoiLVTu+vb8`KSg^@7+0D+lVO z<}Z$eT@U@T!bXa$FO463H>*wZLu&x)fzQN?A|pRw=YpPc>}(xzY>ueARAtr}1gjNL zu_Sm*S$N$|uC7kXyKgac1GH9f*_Bd7k&z$H&SR=~%Sz2_2v{$4bwQnWt^if+H*yt* zo9A>0+#f)i^6uL@-&}$Wxlj0>W`0UWJ;Dxt^|i2{qj_N6rHd8Kns8 zD7Fgv+$hCzqKh7PZEqGo_%7Y>vWYLV@;UE`x?yI1@<{?z_m-8K#I!GC46NaR3VnBY z{RUR!^wadFFn_Wp$3uZ6R$Aw9rH?dBTgjxWUdv_lUEz=)GcfTug!0 z3aD7vzluv(%543yj0KVfkjSDUfm*|BeFc4P5V@R>`CDQltNZCkw%J+uhA}#@0<7nC z_4E`HM?7e%QNgH1gYV_63XALCkHz28S2rJ2<|R)*XooZzz z+%t%@<~6!|6pwhA9grxB5=T5l^Q+u`I|6pT_WGk3dh%1N=XJ2k0#!{xT%X=7f%>0X z15g22CoFZ&f7r2_mK8Qn-X)hyxi|8&-{Dh0Dj<&zt^n(~-je|ws6CC_Q-77{8C9F( zRASd-voejD?@N2_pkwQ2F@Q{K2fzA;6%8LHyxhk%Z^lwypW;f4!a)QELOUNvuQ(p2 z-$?Lc2cq8B0U5C8+%B?nb9H>ekU=b-+CoMKpcW1uVyWk9oqsM%e)=h$izB1-`D&EsI^Rt)OIa+`LND&84Bm6f^b{?*RA>cgy>n{||>Lsv(wd&4! z#}-i4i&I`2Z`jHzoj!w=(NCmi<`<~6{2 z609}PIGxp*I8L_+qKq1Pp}DUTme>3Gh}a7YTW{XSy7+K>7W5k+3?4H-rPN6%n=Sr8 zLHLFt1CbXLb|8ju<|V*Zfb|^j$N;{G^YvihPonw_lwvL{MA9f!+j{K_K$MObZ!xKh z%H_ovy}aOTJx!dg&HP}FY8x&RAcngJ8W~JlgTN|k#iQkiEPVgSn2mmUEa296({(NFp8IUE7()&TnD({*; zQu7lTj2y&M|2eK_WPj)cfGz*@Gm%@lHYVenrHW?t8dy2IzCW9^l)hIg3kqZ}VZbdc zffh?^2fqA@?USh9aXo;qJw72(_FUXkYN68Sf1=Zhk!;3rH&u(L`0K7U)TP>MF%P4h zV_a1OwgRkYdy83v02`nO*ukGyNEaX+{Bw=LUpVMqcJSw4gn)))wC>Dldd8;?fDGfC z`Pui-D25H>gvC+YFY(^NLLfaz{b$6FJf%*W5FW6fR1L#se5*362f@l&i~wfTE0qD5 zy6|zAUCNS1KBU6|j_&-vVucYH3Yt%AvLNG3Z?F+_9ax8&?atuyv?`u32gnY9G<36K zBY6?9L9phWeZCW$*>N8LSn|l>I=^TS?TtX&fnBvvo#4D^^cYqyz!+FWRMJ}8)1xQ* zQ{uNz(}z)8$6;dd~ig>Ijlkcu#o^(E5mVLJbr@doZ%}RU`6v~6^G2{ncse? zXx5ywya`rJCiP`wfD{oUhyW}!Ay~-NN#hN{pWnAX`;*?TcXfVA}8*{uBCIbr7tKQ&rd05*dBTg4ad^UUvJ z0IX&JLn~`fw4hFa)-V^D2MPdv ze4lGs{PkkRQ#tAC(y7y>?%^fC#spZgs3~KgX6Q?5%?e;f$io5{6-id$Pkwj=wSM&d zT3L`8`N50Yd#@I6Ss{+3`|c4go=V0|WQW$RkBHd30@xr}v(Gv&qKQQ7qGRjVXX&Ij zwFaG-cXNMYi``P8)ZxI>ADieU?tAiSLtp80=je(SfNamz*GkYgq`3l{Ra-&K(EYi`F1hFEsOm!fO9g^OhB^ zOQJTEXz^4;F*yw$4%nB?0h>>{-HDB1wq{x1#}HU~Ve-Ly!T__{w21@zv&4b7=!PNT zNPMUzT;GR-MQc_d(~dXWaAstIO@$bRi>G&Hvob26!WD{Ec`^GO`A;TRhjuqI6q@;W z>T?w7{j}Ew!MZyJm=(bTU`Zwb%af?c;Y$VoF-A|x$k&082L<(rRlzKw_9s==Io-s5 z9bCQ6$j^Ryx3*wd<$?>b;S`C#t`CpX%F5_;DWT#}Rt{PDRX<10tzt2nhp2y%arISZ z-Q8JVyoflciZL1geb2C0nt#_|cHk$mCsAtL@t5~x(63s{4RHU`%*;ao-Rjkm6}x$<%4m^A=sV+83P9lJ6T#8o9pZ zC6VOc(rL(m`agWFXx3N)s|dhkl3)gY=-{%CKV@e3Q$H8Imzf`I3T%#A?WuU8jUxgK z6m~@~Ody^6rReRY#vvA*Vj>#(5Iq6h_r&OkyA7g&#UxlUxwOeIQ6lwEl4Jn{UsxCv z4qAim+jDhq&T{bv4R;k}qRCEuZVan^er3G}(1D$ksdy9t2D%?5Q>Is9V$u7Hj8io- zHkOqk78IA*(9y6USa)@b9T}hkEN!wez;dtZEnQ%wz48*PWdQzS0tSjJt!@OH?769* z)EZ@dK2PTd8$v{r;zo!Ca>1?Qi5HQ#_P3l9KwA2r*{;Q67C?g^12SM&C;hjISVmpN z1L{8ajA3PpMJ=A9e={mvxG0{|-<%;lBiVb@YTu4d1O4U_xgmuK@Z&d%W{pL#;>gORV$1OO{zpIQ#Z7s1l;Lr2lt3JPGL?7DcI~*_kTG;4 zaO5BO>1UbCgvVk7r92|0k=>3(B=#b$L&~T^44HkfLV9OC|G%0$pm1z zq$y>ehbP@OfF%p)Hkl-gj^zc*YtvcDe~f{{bu8hA>x6V^ z2^De0uT6D44!ikkxa?a+v+nE|yI@5UvObz?2rz(ER`VrXaT#g3k#=(%(e+LP7A%@f zn^~lHTTbsL6umHx9o{APQcy$_jo_jnSa)=YZLk7hsgLRbhQ-ghQU8sCR=LW2d#3L3 zDy}(9dQzsaGP=)!tID<(qEfIy;6O}UGVs5R{FnBgmcH=8F+P);X)Tt3kg4VE!;iq0QR^B^|>y^ zX0S4BJha4Qa%1Jw&Cf%?dY0|`)M(LyA3507IC}Sgv6R0I60Kte_XWtbhWqE<)R#0^ zvE>|(NlG#WDlRUd4ueyRYjv**h82rDiFA9?jn`Sukv|1{`(0khiR8r6VUz|mTK=7) zS$DLz2UhIpjJIbQ3iv~7*R%a9cqLynkR2BDU!e(upo%U8?P5Co=T}dTugb07J)dKtVTZtz`ETgSmU}}!0J73 z{jJh?R9FtiCEW%=@AS^brm6qGxw;(TzPqpJWqLmqs5MQ1dJsDDw(y{dbYb|t39vOa zuOO=a4w?WKIfpLAoUIIRqZK|sjvZXP&T6PbO+-I=!L!of;E zb_^_AzT``Vd5H-kWnCA=@z;avZwehyOWmSDAI& z8TMpAojZ)ds`>>eptWjuy@1u0eEY~~B4tYgMH(%TS!J6@$7S8eAJ$#2vUFIW4vfyx zZ?;Gruv5(?L8Lhm0fkq`9REu|25dW{v#l8G+;OWRvl{uuLa4xvC-7>2hh8=ifED09 zxcny_>t;-yOq-*fuuEvv+<|v`;s{j$JK0pjhMSa{ll_zzv<;62JN}m+6wSKT0kEbR z6j&mD!EkW|S^(FqeIuG|=rqWM^1vb;*PsE*q$5?t0Ms0h05w#@W=05D)rU4GvDjn9 zT$*?Rtf4>xn*g-6NHRFVx;XO9j{DI2dJ(9hfQ6)~McN-HlHHRCISJj+A{*HYV=BOU zx_udtLb_FJTW$IU(9o)>-LbuG)-_8w2kHQC9Q{_H$ebo8YpncLVqH3syv({yR;V)I z^iEdu3i~-UYYP47!D`QMs6RZZL1!WELMqbeU%Lzg-wIIg@PtMcS%A!Gujn-RVfr=h zvc*eTJmuv$f#2^BcOS4WBowa!JodD!$%-Ep&AP=VSOHWcu-f$-F0lYCq=FsR-TTNB ztmGs8UxpC@PF0M_G5{+IsBGGBFl^}fu3b2AWeQ89xqx*6+Wjb8>0wP(_idX7!FrlK z228or1gtjwiUNW!(uG~PrUQUQ4cbjzXwhq`avTRM%`w!>=D@oy9gz>p3;pW^>w>|n ziglZ=RGHN#sHz&QS^#lo0rOfMjYl(BwK?IvZqojVf4`iHPUAPrL^mR5-2iZr!)UN> z2v|Fc#%VY%<<8rTWx#3n{MrgCa536f0B!O=ZqTud0@mA|xJ^`iWKDb3gjwb0z`HK$ zNksqJ)&E>7EdGLb8QjvkO;;{cH0$P5?STqlO(iLyHM@SDL_q!N#tXf>zuZDQ8I!Cp zx8_i@RSTAQM~e!hTPA0n^OVHE^g>}w-(w?OtRNT zV-7fp;z%xVDMkO|0Ngl|3JFxWioED;G=-ggZnC}ZQ3Gq}1VWDqRF=`Qs#s{J=T_Bo0P7|M7+0q2 zQZRW6E1dkAt#kyzy6Kb-wj9+W&C_ts+Y$LFg)k@PEhXR{-nlCxA0ijkfBiMmWR=@K z6Nl3k-A^bMY#wC6g32O?^GO2Ol>zX}Wx&R^?Pb?qdVIn~ z0B#4CNC1wss#I?Z)`eUEcuVO?ib!irS?J31ET?96?19yMtrQ7GIU)Uj$mU5Ty}*GB zPWt^P-K7dxUyalyzYcz*2G-C>E3A)tk4Id@23eOjw41DZ z(jO+wI>{!pl3&iO8>$$rp#ZAYswIo-1f8q2MC_UZ+$AjOsw=1fY^ek5&KT0H8KWO# z$utkUhSR6~N6HH^F7o4~H8TNgzD=;gZvd>~zf|Ys#H@Vr7cM$=VDZGb@uYX{vaa@i zSg4)-^zSrS#TD((`4@Lge4aM1^Mx>fgX+iStPCd+X?VM=+w2ehFDT}dUbPR_p95fR z<4v$?L7}j22L{>T1nXj9ia-1xZ;KVtzg{!|wc*Uybf{$WpE0HIJu7qC&tgVkOu&P-(COL_v{_{qb()T8QJc=3z?1-uXEMm3->W;nxIq1^v z*1tFo#@lxF^_Ek`eXxVehDgO3wF|CH4*_%dN^G{i4s0IUjJyyM}d0jzm8!P??z z1S^uBKKy>m>KThsMuo$%#JFaQ6HUO%0u}Msx%%%clO7ak4PB1S-K@XI)eMl>;Gy8P z%eu~jwUz$Qp*y*=qTve(&K&auPqLLAqTdtBpE#Mb4+qJtY2g7o{1Gdcwmw&YI+fLG zUHPF!g+44{+ixBaWYo}`HF{$j^`P|+Przh$*-X6;2QC$^iQO(VWOA#?I}uBZH7gX& zx-N#niiL(w0?Sw#3X_7FZ|WAYDgkQ;sF*K+)M%Fek9+)CRLnMY>&fu1lLk^iffm=P zBWd0)LauXYv&+g7t>YkA*Tx`Nk45ox^G*B*3vOivv)hLgq_0ta)>efl}8)Y$qwi369qE+(uCg4l9ZAKOpHTB^*twqxs|fDG8|XMRX}z zIAE2sAC^hQUIAER{kU)Le;osC)-z8U$`=Kw{iPN;W5h6)O5oR8$)d$<=iT>7<-m)B zTdrNHXjZR-6+1SA7zjs=m7#dTZU(3#H4(sKGl;|!E(XlYsI>-thtojD5c7$P=Na}( zx8FQadeh3>I~rqyU|rL~o6L#@0F&13_Ycc_fQ|BJyvYkFGhJ%wBxC>w?3>eAZOvJ7e=ew1$%47IW!ma}anDPq}LtqU-#7r~08r}q5^18A%aOq4tN zSw}z`datSkY@`4?X~{fdYyDpx-2D($h`^a{iArQch6lyPo5y) zcRx7FwE_YfxjyO2NS}fUPugT4*icq;*Xea7k6CMYP_VJP`Bgu!-MSUaB#kT*i>S{?#$lctA^=#EH>VpGp?gTn z@1>tuwAMkeHm^Fc)@am#YpoPw1S^9s%^5wG6}}&N@f7YJGO>jE0jOl&ut#55R14>j zynz;%F>L(fNpEWnS|04`gr=8?$BD76*Q?B01z?39tV^1--aU6(&IKUNoACxKpha^d z(P}KIRLF^FWNpA;d6^BAIqE^0yS~k2l!!-;n&QoJ8bMCbPAz(cv zs<9~`y-Nrfg;W&a=sL7xrzy|^;x4)L)MInEhB@pa4Vm#ED||wz5X{vv#x3u9k3>KzmR1SFmT*jSt%7yIZuwI z_RSirp%Hy96APZefQ=fAeM%QQKHGFfZ{ZjKqUK-dL6k?Q1_0S?=>mEaEK%ogeHVFv zr4u2f8=PFl3m3EPcMcJb*$V^PY={zAdBHMj7=xCwGHAbD!$!GQJn=$~7Em;nJrgF= z5&t29)EH#=e2)a{rCf6rfnHy&13Unc8NhJDATKfoNn-~7tvB^|Eypr3ijIv6SOE&? zCo3&@gmfvGCGYh&MB|ql2e751kh{jZkV7o_6h}VkWK|=l-SSVCIdYipo|~??R*L-_ zzcRs4ONO3q&6mzMY?nm>FtfNJIp8l=tOBejwRJ6KzBqLJhchq}z~Nf))bF1xeaKxx za|DCsJo$_+mL_k{AU3Z2t}H;k_H?yhUgp zp@W)#xxf(oU}~Hiv0%s>*Q|fI2-cxN6lpX`_8a9$0bg6hsqv}3=$laE#j(Ebw zf>WW@FW>3b6u{p3z)0b>PVRmYYy8T1-S}+M!`c(YEN=iUg99tTdZK$W z0P!|_ZnUXZh-_bP!4UTQ>FC#A5Hy0I@F^% z%7EO=nHl&oejTx^TeX95lEMs(I#`_qEEZ6_cv{8_p-2`38zIA>ld$3PYotzs<6oVq zJJP}Ot4|BxO~8h1@zp~9E8|*=1YlUKK=EYGTviEf_R0GU>>6s&nQPDOZgpw!a{#XO%I|gK zi>PI_?Y6;|elqSg-Y_T%Wuyph#KHfyG59y%5T4~N19fpg+sy$P@Y@M(T)d63>qZ=~ z#u}4r*ccwwYG!UUNpv*UoT=c4Zd9lU@ei{jF2StNnU6Tn|hOk4i`><0|QUlyh_omRRLDS zF9KKq7a&dTeYNG3ajZf|=FyK64;XOIM%2mB+Sc0#GaQ?uW;V~Y=XDpBCT6igGeegG=z1B6 zW0gavK0jrP%B)ogR?e_t0Bbc_w2*fEYlJXES@{e7yv?sa=$?K-XKuZ9kg%ILbQ3}-)9H|!M1+HbVEwge>j+&qcUj}-$CzqZ zqq~&TtAk)tS-mD2aa^0$thoWmu#kJE&lGjT();&i^&ipe!%f%sjVN}rJmru^&l&fU za9kK@Hb+RPKTfr-4Ww>nbMUEKRx6tI*CtgDR?LMjghgYY7f&y~#Pa9ZE1s-!*Wl6N zyf><~HRp7bV)5pmeL$e7+)aSC{-bSVV1S%x_01w(T>MEC$qJeF9KZ^8HIiCSvY(&2)ni}{2UuP_O`FAkjuaKi&Q*H65y-Gn)|`Eju&-m2liA6?>AMO0s5q06 z651PP-ZzM4Qaej)cyzq|U&Dps3kOs&hmTk0~z% ziGn+3v0CbAb0T(ttnt6~pxkoXAhvJ5{s=a^CuB`sgO=Osb1t$yr}P1RS%XLP zED8<3@jAVHQDa12PvS>^vSHZy)SDJ%XD4zk>f^MnYZT48@_6roRdf<=jB7e|601Xv ze67jmsqODS>Og_M=(Wtj?_2 zZ2hZ>C`MPEbrIM7_}o;ZQVVd~P<K#J(T{*CwIhnWO(!ncXi;WZCrNrlmoCkbX6@ zSOqHPQvue7KC`A=LMS{Dmxc$_(K-cy(aAo$D3N`g6nT4c2&S-nUwy|C8srQ)jXrkLtf5&1^w4Ztce! zJ*NNetF2s0=)ZfwtHAJLTtTq@)S!BUH4`8st?8DnAPEqy*BjlMLsk22zbnA3)g7!Y z-hYd#_{iSew_3zvdfR^8HDZ51z{*#3{yYS%&8l-ASo>WYT$}%nM@O@qDf%x&bM1yl zsbj1W*Q`I)uijuiW!X}pZ+M9W2Pn|Wes}IIt4Lxv8m?)-I{?<@)f}v;mv(1O$2}|b zrNc4{znA*6TZZVr+FiznKUT>=2Oq2zP^d+1C-f+_J<>?QQaEp+&!~|m&)~WWux6d) zpj+RSp8@AH&>?%HqFI;wV0AvbzDGwh5CrQoAFPfh)_3N720E(Dx-8R|0nVq__vmB> zI_3mqz_QE}o%HMb@-yIg20G?!QZ(z2K3E-3uJ6#<40Q6r>TE)NZ|-HF0<0On3~(>a zzGo*g&?#4y0U0Ma;m`NsXTY@#bjsbVXx1Nmu)3CH-?1|p=o|!Vy3ef6q}2D~Rt7rn zQJK{Tt6Mqt{W_F^5U@6N$ffVb&wx7_=)7l(qFKMM=Y!Rq6#HJC%D|a>eXu%}OW%*n v8SugCa)N#L&SfA7R_9#$p8O2Bmx2ESYsQK!_cfR<00000NkvXXu0mjfb|YRt literal 0 HcmV?d00001 From 6adaaad91cf4a1e669414d8dd6e470377c9cc0a2 Mon Sep 17 00:00:00 2001 From: cjames23 Date: Mon, 26 Jun 2023 08:01:48 -0700 Subject: [PATCH 17/23] Add http to selective tests for amazon. --- dev/breeze/tests/test_selective_checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4869d70739100..c786b8ac1faba 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -353,7 +353,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/file.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google imap " + "common.sql exasol ftp google http imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -367,7 +367,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp," + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp, http" "imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", From d75fa82dede99662e8158775b9ea8bd2e9acabd0 Mon Sep 17 00:00:00 2001 From: cjames23 Date: Mon, 26 Jun 2023 09:01:38 -0700 Subject: [PATCH 18/23] Fix amazon selective checks --- dev/breeze/tests/test_selective_checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index c786b8ac1faba..24e0091611aa6 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -311,7 +311,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/__init__.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google imap " + "common.sql exasol ftp google http imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -325,7 +325,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "run-amazon-tests": "true", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,imap," + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap," "mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", From 595dc04a3fc18f4453d4aba91b97ddea402ce39c Mon Sep 17 00:00:00 2001 From: cjames23 Date: Mon, 26 Jun 2023 10:08:18 -0700 Subject: [PATCH 19/23] Add amzn to affected providers http. --- dev/breeze/tests/test_selective_checks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 24e0091611aa6..eea56f2433916 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -191,7 +191,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "tests/providers/http/file.py", ), { - "affected-providers-list-as-string": "airbyte apache.livy " + "affected-providers-list-as-string": "airbyte amazon apache.livy " "dbt.cloud dingding discord http", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -205,7 +205,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Always " - "Providers[airbyte,apache.livy,dbt.cloud,dingding,discord,http]", + "Providers[airbyte,amazon,apache.livy,dbt.cloud,dingding,discord,http]", }, id="Helm tests, http and all relevant providers, kubernetes tests and " "docs should run even if unimportant files were added", From 3fdaa4d37d4ccf9c41f3d986ee5fdfc010995aef Mon Sep 17 00:00:00 2001 From: cjames23 Date: Mon, 26 Jun 2023 10:22:11 -0700 Subject: [PATCH 20/23] Fix missing comma in selective checks --- dev/breeze/tests/test_selective_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 233d6b0547ad1..babedc7667926 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -367,7 +367,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp, http" + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http," "imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", From 0fa452596933ee5f75a636deeb7d91567047a2fa Mon Sep 17 00:00:00 2001 From: cjames23 Date: Mon, 26 Jun 2023 10:53:36 -0700 Subject: [PATCH 21/23] Selective checks http change run-amazon-tests to true expected --- dev/breeze/tests/test_selective_checks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index babedc7667926..32fde1816e9b8 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -200,7 +200,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", - "run-amazon-tests": "false", + "run-amazon-tests": "true", "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", From 1f06a95d5df0a389528f53d081d88377efa4ebaf Mon Sep 17 00:00:00 2001 From: cjames23 Date: Mon, 26 Jun 2023 15:29:17 -0700 Subject: [PATCH 22/23] Fix Selective Checks for Amazon http dependency --- dev/breeze/tests/test_selective_checks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 32fde1816e9b8..f41155f5e0a63 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -311,7 +311,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/__init__.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google http imap " + "common.sql exasol ftp google imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -325,7 +325,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "run-amazon-tests": "true", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap," + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,imap," "mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", @@ -353,7 +353,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/file.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google http imap " + "common.sql exasol ftp google imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -367,7 +367,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "run-kubernetes-tests": "false", "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http," + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp," "imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", From 8a5bc35809d7883464f35ade93a225e67408537a Mon Sep 17 00:00:00 2001 From: cjames23 Date: Mon, 26 Jun 2023 15:40:39 -0700 Subject: [PATCH 23/23] Fix Selective Checks for Amazon http dep --- dev/breeze/tests/test_selective_checks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index f41155f5e0a63..0081b6a2a795b 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -204,8 +204,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "docs-build": "true", "run-kubernetes-tests": "true", "upgrade-to-newer-dependencies": "false", - "parallel-test-types-list-as-string": "Always " - "Providers[airbyte,amazon,apache.livy,dbt.cloud,dingding,discord,http]", + "parallel-test-types-list-as-string": "Providers[amazon] Always " + "Providers[airbyte,apache.livy,dbt.cloud,dingding,discord,http]", }, id="Helm tests, http and all relevant providers, kubernetes tests and " "docs should run even if unimportant files were added", @@ -311,7 +311,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/__init__.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google imap " + "common.sql exasol ftp google http imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -325,7 +325,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "run-amazon-tests": "true", "parallel-test-types-list-as-string": "Providers[amazon] Always " - "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,imap," + "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap," "mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", @@ -353,7 +353,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): ("airflow/providers/amazon/file.py",), { "affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes " - "common.sql exasol ftp google imap " + "common.sql exasol ftp google http imap " "mongo mysql postgres salesforce ssh", "all-python-versions": "['3.8']", "all-python-versions-list-as-string": "3.8", @@ -368,7 +368,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "parallel-test-types-list-as-string": "Providers[amazon] Always " "Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp," - "imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", + "http,imap,mongo,mysql,postgres,salesforce,ssh] Providers[google]", }, id="Providers tests run including amazon tests if amazon provider files changed", ),