From a274ceab497533a04bd2dcb43b7ab77c3b1a7b74 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Fri, 13 Oct 2023 19:25:52 +0530 Subject: [PATCH 01/36] Add Cohere Provider --- .../airflow_providers_bug_report.yml | 1 + CONTRIBUTING.rst | 4 +- INSTALL | 3 +- .../cohere/.latest-doc-only-change.txt | 1 + airflow/providers/cohere/CHANGELOG.rst | 22 +++++ airflow/providers/cohere/__init__.py | 24 +++++ airflow/providers/cohere/hooks/__init__.py | 24 +++++ airflow/providers/cohere/hooks/cohere.py | 72 ++++++++++++++ .../providers/cohere/operators/__init__.py | 16 +++ .../providers/cohere/operators/embedding.py | 82 ++++++++++++++++ airflow/providers/cohere/provider.yaml | 52 ++++++++++ .../commits.rst | 16 +++ .../connections.rst | 34 +++++++ .../apache-airflow-providers-cohere/index.rst | 98 +++++++++++++++++++ .../installing-providers-from-sources.rst | 18 ++++ .../operators/cohere.rst | 57 +++++++++++ docs/apache-airflow/extra-packages-ref.rst | 2 + tests/providers/cohere/__init__.py | 16 +++ tests/providers/cohere/hooks/__init__.py | 16 +++ tests/providers/cohere/hooks/test_cohere.py | 38 +++++++ tests/providers/cohere/operators/__init__.py | 16 +++ .../cohere/operators/test_embedding.py | 86 ++++++++++++++++ 22 files changed, 695 insertions(+), 3 deletions(-) create mode 100644 airflow/providers/cohere/.latest-doc-only-change.txt create mode 100644 airflow/providers/cohere/CHANGELOG.rst create mode 100644 airflow/providers/cohere/__init__.py create mode 100644 airflow/providers/cohere/hooks/__init__.py create mode 100644 airflow/providers/cohere/hooks/cohere.py create mode 100644 airflow/providers/cohere/operators/__init__.py create mode 100644 airflow/providers/cohere/operators/embedding.py create mode 100644 airflow/providers/cohere/provider.yaml create mode 100644 docs/apache-airflow-providers-cohere/commits.rst create mode 100644 docs/apache-airflow-providers-cohere/connections.rst create mode 100644 docs/apache-airflow-providers-cohere/index.rst create mode 100644 docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst create mode 100644 docs/apache-airflow-providers-cohere/operators/cohere.rst create mode 100644 tests/providers/cohere/__init__.py create mode 100644 tests/providers/cohere/hooks/__init__.py create mode 100644 tests/providers/cohere/hooks/test_cohere.py create mode 100644 tests/providers/cohere/operators/__init__.py create mode 100644 tests/providers/cohere/operators/test_embedding.py diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml index 4c3a0a98194c7..e8a221be9cd42 100644 --- a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml @@ -48,6 +48,7 @@ body: - celery - cloudant - cncf-kubernetes + - cohere - common-io - common-sql - daskexecutor diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index a3fe3386eb3b0..7abf7ad2b9dab 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -671,7 +671,7 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, common.io, common.sql, crypto, dask, daskexecutor, databricks, datadog, +cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, @@ -1444,7 +1444,7 @@ You can join the channels via links at the `Airflow Community page `_ for: * detailed discussions on big proposals (Airflow Improvement Proposals also name AIPs) * helpful, shared resources (for example Apache Airflow logos - * information that can be reused by others (for example instructions on preparing workshops) + * information that can be re-used by others (for example instructions on preparing workshops) * GitHub `Pull Requests (PRs) `_ for: * discussing implementation details of PRs * not for architectural discussions (use the devlist for that) diff --git a/INSTALL b/INSTALL index b76af9d62200d..f060a49324b8f 100644 --- a/INSTALL +++ b/INSTALL @@ -98,7 +98,7 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, common.io, common.sql, crypto, dask, daskexecutor, databricks, datadog, +cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks, datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, @@ -108,6 +108,7 @@ oracle, otel, pagerduty, pandas, papermill, password, pinot, plexus, postgres, p redis, s3, s3fs, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk + # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/airflow/providers/cohere/.latest-doc-only-change.txt b/airflow/providers/cohere/.latest-doc-only-change.txt new file mode 100644 index 0000000000000..14a9c1ef7c65f --- /dev/null +++ b/airflow/providers/cohere/.latest-doc-only-change.txt @@ -0,0 +1 @@ +c645d8e40c167ea1f6c332cdc3ea0ca5a9363205 diff --git a/airflow/providers/cohere/CHANGELOG.rst b/airflow/providers/cohere/CHANGELOG.rst new file mode 100644 index 0000000000000..450e8ed420cd7 --- /dev/null +++ b/airflow/providers/cohere/CHANGELOG.rst @@ -0,0 +1,22 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. diff --git a/airflow/providers/cohere/__init__.py b/airflow/providers/cohere/__init__.py new file mode 100644 index 0000000000000..c88b314548bd6 --- /dev/null +++ b/airflow/providers/cohere/__init__.py @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY +# diff --git a/airflow/providers/cohere/hooks/__init__.py b/airflow/providers/cohere/hooks/__init__.py new file mode 100644 index 0000000000000..c88b314548bd6 --- /dev/null +++ b/airflow/providers/cohere/hooks/__init__.py @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY IT, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/provider_packages` DIRECTORY +# diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py new file mode 100644 index 0000000000000..456c0956919f9 --- /dev/null +++ b/airflow/providers/cohere/hooks/cohere.py @@ -0,0 +1,72 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import Any + +import cohere + +from airflow.hooks.base import BaseHook + + +class CohereHook(BaseHook): + """ + Use Cohere(https://docs.cohere.com/docs) to interact with Cohere platform. + + :param conn_id: :ref:`Cohere connection id ` + """ + + conn_name_attr = "conn_id" + default_conn_name = "cohere_default" + conn_type = "cohere" + hook_name = "Cohere" + + def __init__(self, conn_id: str = default_conn_name) -> None: + super().__init__() + self.conn_id = conn_id + self.client = None + + def get_conn(self) -> None: + raise NotImplementedError() + + def _get_api_key(self) -> str: + return str(self.get_connection(self.conn_id).password) + + def get_client(self) -> cohere.Client: + if self.client is None: + self.client = cohere.Client(self._get_api_key()) + return self.client + + def embed_text(self, texts: list[str], model: str = "embed-multilingual-v2.0") -> list[list[float]]: + response = self.get_client().embed(texts=texts, model=model) + embeddings = response.embeddings + return embeddings # type:ignore[no-any-return] + + @staticmethod + def get_ui_field_behaviour() -> dict[str, Any]: + return { + "hidden_fields": ["host", "schema", "login", "port", "extra"], + "relabeling": {}, + } + + def test_connection(self) -> tuple[bool, str]: + try: + self.get_client().list_custom_models() + return True, "Connection established" + except Exception as e: + return False, str(e) diff --git a/airflow/providers/cohere/operators/__init__.py b/airflow/providers/cohere/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/airflow/providers/cohere/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py new file mode 100644 index 0000000000000..409d1669a9e73 --- /dev/null +++ b/airflow/providers/cohere/operators/embedding.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from functools import cached_property +from typing import TYPE_CHECKING, Any, Callable, Collection, Mapping + +from airflow.exceptions import AirflowException +from airflow.models import BaseOperator +from airflow.providers.cohere.hooks.cohere import CohereHook + +if TYPE_CHECKING: + from airflow.utils.context import Context + + +class CohereEmbeddingOperator(BaseOperator): + """ + Creates the embedding base by interacting with Cohere's hosted services. + + :param conn_id: Optional. The name of the Airflow connection to get connection + information for Cohere. Defaults to "cohere_default". + :param input_text: list of text items that need to be embedded. Only one of input_text or input_callable + should be provided. + :param input_callable: The callable that provides the input texts to generate embeddings for. + Only one of input_text or input_callable should be provided. + :param input_callable_args: The list of arguments to be passed to ``input_callable`` + :param input_callable_kwargs: The kwargs to be passed to ``input_callable`` + """ + + def __init__( + self, + input_text: list[str] | None = None, + input_callable: Callable[[Any], list[str]] | None = None, + input_callable_args: Collection[Any] | None = None, + input_callable_kwargs: Mapping[str, Any] | None = None, + conn_id: str = CohereHook.default_conn_name, + **kwargs: Any, + ): + super().__init__(**kwargs) + self.conn_id = conn_id + self.input_text = input_text + self.input_callable = input_callable + self.input_callable_args = input_callable_args or () + self.input_callable_kwargs = input_callable_kwargs or {} + self.text_to_embed = self._get_text_to_embed() + + @cached_property + def hook(self) -> CohereHook: + """Return an instance of the CohereHook.""" + return CohereHook(conn_id=self.conn_id) + + def _get_text_to_embed(self) -> list[str]: + """Get the text to embed by evaluating ``input_text`` and ``input_callable``.""" + if all([self.input_text, self.input_callable]): + raise RuntimeError("Only one of 'input_text' and 'input_callable' is allowed") + if self.input_callable: + if not callable(self.input_callable): + raise AirflowException("`input_callable` param must be callable") + return self.input_callable(*self.input_callable_args, **self.input_callable_kwargs) + elif self.input_text: + return self.input_text + else: + raise RuntimeError("Either one of 'input_text' and 'input_callable' must be provided") + + def execute(self, context: Context) -> list[list[float]]: + """Embed texts using Cohere embed services.""" + return self.hook.embed_text(self.text_to_embed) diff --git a/airflow/providers/cohere/provider.yaml b/airflow/providers/cohere/provider.yaml new file mode 100644 index 0000000000000..9483bad21df31 --- /dev/null +++ b/airflow/providers/cohere/provider.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-cohere + +name: Cohere + +description: | + `Cohere `__ + +suspended: false + +versions: + - 1.0.0 + +integrations: + - integration-name: Cohere + external-doc-url: https://docs.cohere.com/docs + tags: [software] + +dependencies: + - apache-airflow>=2.4.0 + - cohere + +hooks: + - integration-name: Cohere + python-modules: + - airflow.providers.cohere.hooks.cohere + +operators: + - integration-name: Cohere Embedding Model + python-modules: + - airflow.providers.cohere.operators.embedding + +connection-types: + - hook-class-name: airflow.providers.cohere.hooks.cohere.CohereHook + connection-type: cohere diff --git a/docs/apache-airflow-providers-cohere/commits.rst b/docs/apache-airflow-providers-cohere/commits.rst new file mode 100644 index 0000000000000..106592bd11775 --- /dev/null +++ b/docs/apache-airflow-providers-cohere/commits.rst @@ -0,0 +1,16 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. diff --git a/docs/apache-airflow-providers-cohere/connections.rst b/docs/apache-airflow-providers-cohere/connections.rst new file mode 100644 index 0000000000000..bfbc8b0ce0cf5 --- /dev/null +++ b/docs/apache-airflow-providers-cohere/connections.rst @@ -0,0 +1,34 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/connection:cohere: + +Cohere Connection +======================= + +The `Cohere `__ connection type enables access to Cohere APIs, which we can use to interact with multilingual models. + +Default Connection IDs +---------------------- + +Cohere hooks point to ``cohere_default`` by default. + +Configuring the Connection +-------------------------- + +Password (required) + Specify the password to connect. diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst new file mode 100644 index 0000000000000..084c15cd29fca --- /dev/null +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -0,0 +1,98 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-cohere`` +====================================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Guides + + Connection types + Operators + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + Python API <_api/airflow/providers/cohere/index> + PyPI Repository + Installing from sources + + +Package apache-airflow-providers-cohere +----------------------------------------- + +`Cohere `__ + + +Release: 1.0.0 + +Provider package +---------------- + +This is a provider package for ``cohere`` APIs. All classes for this provider package +are in ``airflow.providers.cohere`` python module. + +Installation +------------ + +You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below) +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-cohere`` + + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider package is ``2.4.0``. + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=2.4.0`` +================== ================== + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits diff --git a/docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst b/docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..1c90205d15b3a --- /dev/null +++ b/docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-cohere/operators/cohere.rst b/docs/apache-airflow-providers-cohere/operators/cohere.rst new file mode 100644 index 0000000000000..68b5ce41f55b2 --- /dev/null +++ b/docs/apache-airflow-providers-cohere/operators/cohere.rst @@ -0,0 +1,57 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/operator:CohereEmbeddingOperator: + +CohereEmbeddingOperator +======================== + +Use the :class:`~astronomer_providers_llm.providers.cohere.operators.embedding.CohereEmbeddingOperator` to +interact with Cohere APIs to create embeddings for a given text. + + +Using the Operator +^^^^^^^^^^^^^^^^^^ + +The CohereEmbeddingOperator requires the ``texts`` as an input to embedding API. Use the ``cohere_conn_id`` parameter to specify the Cohere connection to use to +connect to your account. + +An example using the operator is in way: + +Example Code: +------------- + +.. code-block:: python + + from datetime import datetime + + from airflow import DAG + + from astronomer_providers_llm.providers.cohere.operators.embedding import CohereEmbeddingOperator + + with DAG("example_cohere_embedding", schedule=None, start_date=datetime(2023, 1, 1), catchup=False) as dag: + texts = [ + "On Kernel-Target Alignment. We describe a family of global optimization procedures", + " that automatically decompose optimization problems into smaller loosely coupled", + " problems, then combine the solutions of these with message passing algorithms.", + ] + + def get_text(): + return texts + + CohereEmbeddingOperator(input_text=texts, task_id="embedding_via_text") + CohereEmbeddingOperator(input_callable=get_text, task_id="embedding_via_callable") diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index dba750908d853..9f7fcc9a03480 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -172,6 +172,8 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | cloudant | ``pip install 'apache-airflow[cloudant]'`` | Cloudant hook | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ +| cohere | ``pip install 'apache-airflow[cohere]'`` | Cohere hook and operators | ++---------------------+-----------------------------------------------------+-----------------------------------------------------+ | databricks | ``pip install 'apache-airflow[databricks]'`` | Databricks hooks and operators | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | datadog | ``pip install 'apache-airflow[datadog]'`` | Datadog hooks and sensors | diff --git a/tests/providers/cohere/__init__.py b/tests/providers/cohere/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/cohere/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/cohere/hooks/__init__.py b/tests/providers/cohere/hooks/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/cohere/hooks/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/cohere/hooks/test_cohere.py b/tests/providers/cohere/hooks/test_cohere.py new file mode 100644 index 0000000000000..9d8dd4f088df9 --- /dev/null +++ b/tests/providers/cohere/hooks/test_cohere.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest.mock import patch + +from airflow.models import Connection +from airflow.providers.cohere.hooks.cohere import ( + CohereHook, +) + + +class TestCohereHook: + """ + Test for CohereHook + """ + + def test__get_api_key(self): + api_key = "test" + with patch.object( + CohereHook, "get_connection", return_value=Connection(conn_type="cohere", password=api_key) + ): + hook = CohereHook() + assert api_key == hook._get_api_key() diff --git a/tests/providers/cohere/operators/__init__.py b/tests/providers/cohere/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/providers/cohere/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/providers/cohere/operators/test_embedding.py b/tests/providers/cohere/operators/test_embedding.py new file mode 100644 index 0000000000000..9302d112bdf07 --- /dev/null +++ b/tests/providers/cohere/operators/test_embedding.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from airflow.exceptions import AirflowException +from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator + + +@patch("astronomer_providers_llm.providers.cohere.hooks.cohere.CohereHook._get_api_key") +@patch("cohere.Client") +def test_cohere_embedding_operator(cohere_client, _get_api_key): + """ + Test Cohere client is getting called with the correct key and that + the execute methods returns expected response. + """ + embedded_obj = [1, 2, 3] + + class resp: + embeddings = embedded_obj + + api_key = "test" + texts = ["On Kernel-Target Alignment. We describe a family of global optimization procedures"] + + _get_api_key.return_value = api_key + client_obj = MagicMock() + cohere_client.return_value = client_obj + client_obj.embed.return_value = resp + + op = CohereEmbeddingOperator(task_id="embed", conn_id="some_conn", input_text=texts) + + val = op.execute(context={}) + cohere_client.assert_called_once_with(api_key) + assert val == embedded_obj + + +@pytest.mark.parametrize( + "inputs, expected_exception, error_message", + ( + ( + {"input_text": "test", "input_callable": "some_value"}, + RuntimeError, + "Only one of 'input_text' and 'input_callable' is allowed", + ), + ({"input_callable": "str_type"}, AirflowException, "`input_callable` param must be callable"), + ({}, RuntimeError, "Either one of 'input_text' and 'input_callable' must be provided"), + ), +) +def test_validate_inputs(inputs, expected_exception, error_message): + print( + "inputs : ", inputs, " expected_exception : ", expected_exception, " error_message: ", error_message + ) + with pytest.raises(expected_exception, match=error_message): + CohereEmbeddingOperator(task_id="embed", conn_id="some_conn", **inputs) + + +@pytest.mark.parametrize( + "inputs, expected_text", + ( + ({"input_text": ["some text", "some text"]}, ["some text", "some text"]), + ( + {"input_callable": lambda: ["some other text", "some other text"]}, + ["some other text", "some other text"], + ), + ), +) +def test_get_text(inputs, expected_text): + op = CohereEmbeddingOperator(task_id="embed", conn_id="some_conn", **inputs) + assert op.text_to_embed == expected_text From feea2406b0413c2122d7fe5514c036ead7361368 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Fri, 13 Oct 2023 19:38:45 +0530 Subject: [PATCH 02/36] Add Cohere Provider --- docs/apache-airflow-providers-cohere/operators/cohere.rst | 5 +++++ tests/providers/cohere/operators/test_embedding.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-cohere/operators/cohere.rst b/docs/apache-airflow-providers-cohere/operators/cohere.rst index 68b5ce41f55b2..78165c91cf7fd 100644 --- a/docs/apache-airflow-providers-cohere/operators/cohere.rst +++ b/docs/apache-airflow-providers-cohere/operators/cohere.rst @@ -20,7 +20,11 @@ CohereEmbeddingOperator ======================== +<<<<<<< HEAD Use the :class:`~astronomer_providers_llm.providers.cohere.operators.embedding.CohereEmbeddingOperator` to +======= +Use the :class:`~airflow.providers.cohere.operators.embedding.CohereEmbeddingOperator` to +>>>>>>> d50e02cd6b (Add Cohere Provider) interact with Cohere APIs to create embeddings for a given text. @@ -42,6 +46,7 @@ Example Code: from airflow import DAG from astronomer_providers_llm.providers.cohere.operators.embedding import CohereEmbeddingOperator + from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator with DAG("example_cohere_embedding", schedule=None, start_date=datetime(2023, 1, 1), catchup=False) as dag: texts = [ diff --git a/tests/providers/cohere/operators/test_embedding.py b/tests/providers/cohere/operators/test_embedding.py index 9302d112bdf07..8703141b776a9 100644 --- a/tests/providers/cohere/operators/test_embedding.py +++ b/tests/providers/cohere/operators/test_embedding.py @@ -24,7 +24,7 @@ from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator -@patch("astronomer_providers_llm.providers.cohere.hooks.cohere.CohereHook._get_api_key") +@patch("airflow.providers.cohere.hooks.cohere.CohereHook._get_api_key") @patch("cohere.Client") def test_cohere_embedding_operator(cohere_client, _get_api_key): """ From 32f5dfe2be9a73210f0860ec8d7816476edd72c8 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 15:51:57 +0530 Subject: [PATCH 03/36] Move link to seealso sphinx directive --- airflow/providers/cohere/hooks/cohere.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index 456c0956919f9..c578f4124d126 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -26,7 +26,9 @@ class CohereHook(BaseHook): """ - Use Cohere(https://docs.cohere.com/docs) to interact with Cohere platform. + Use Cohere Python SDK to interact with Cohere platform. + + .. seealso:: https://docs.cohere.com/docs :param conn_id: :ref:`Cohere connection id ` """ From 3035a471d6080a355233f3247fd7575f83a3fcab Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 15:56:10 +0530 Subject: [PATCH 04/36] Updated check for parameters --- airflow/providers/cohere/operators/embedding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index 409d1669a9e73..aede80441fdb7 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -66,7 +66,7 @@ def hook(self) -> CohereHook: def _get_text_to_embed(self) -> list[str]: """Get the text to embed by evaluating ``input_text`` and ``input_callable``.""" - if all([self.input_text, self.input_callable]): + if self.input_text and self.input_callable: raise RuntimeError("Only one of 'input_text' and 'input_callable' is allowed") if self.input_callable: if not callable(self.input_callable): From 2dcb0bb59de0c3632ad69791dd44348644de9446 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 16:29:47 +0530 Subject: [PATCH 05/36] Update dependency of the cohere --- airflow/providers/cohere/provider.yaml | 2 +- generated/provider_dependencies.json | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/airflow/providers/cohere/provider.yaml b/airflow/providers/cohere/provider.yaml index 9483bad21df31..37427709c9360 100644 --- a/airflow/providers/cohere/provider.yaml +++ b/airflow/providers/cohere/provider.yaml @@ -34,7 +34,7 @@ integrations: tags: [software] dependencies: - - apache-airflow>=2.4.0 + - apache-airflow>=2.5.0 - cohere hooks: diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 38d2259dab3de..2d58848370f9a 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -269,6 +269,14 @@ "cross-providers-deps": [], "excluded-python-versions": [] }, + "cohere": { + "deps": [ + "apache-airflow>=2.5.0", + "cohere" + ], + "cross-providers-deps": [], + "excluded-python-versions": [] + }, "common.io": { "deps": [ "apache-airflow>=2.8.0" From 2476fd6ec8f45217d8c783e9977f2acdcffcb2a7 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 17:04:36 +0530 Subject: [PATCH 06/36] Move the dag out of rst and into system tests --- .../apache-airflow-providers-cohere/index.rst | 6 +++ .../operators/cohere.rst | 26 ++-------- tests/system/providers/cohere/__init__.py | 16 +++++++ .../example_cohere_embedding_operator.py | 48 +++++++++++++++++++ 4 files changed, 75 insertions(+), 21 deletions(-) create mode 100644 tests/system/providers/cohere/__init__.py create mode 100644 tests/system/providers/cohere/example_cohere_embedding_operator.py diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index 084c15cd29fca..aab09cad3ca93 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -53,6 +53,12 @@ PyPI Repository Installing from sources +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/providers/cohere/index> Package apache-airflow-providers-cohere ----------------------------------------- diff --git a/docs/apache-airflow-providers-cohere/operators/cohere.rst b/docs/apache-airflow-providers-cohere/operators/cohere.rst index 78165c91cf7fd..795e3f0329ec0 100644 --- a/docs/apache-airflow-providers-cohere/operators/cohere.rst +++ b/docs/apache-airflow-providers-cohere/operators/cohere.rst @@ -39,24 +39,8 @@ An example using the operator is in way: Example Code: ------------- -.. code-block:: python - - from datetime import datetime - - from airflow import DAG - - from astronomer_providers_llm.providers.cohere.operators.embedding import CohereEmbeddingOperator - from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator - - with DAG("example_cohere_embedding", schedule=None, start_date=datetime(2023, 1, 1), catchup=False) as dag: - texts = [ - "On Kernel-Target Alignment. We describe a family of global optimization procedures", - " that automatically decompose optimization problems into smaller loosely coupled", - " problems, then combine the solutions of these with message passing algorithms.", - ] - - def get_text(): - return texts - - CohereEmbeddingOperator(input_text=texts, task_id="embedding_via_text") - CohereEmbeddingOperator(input_callable=get_text, task_id="embedding_via_callable") +.. exampleinclude:: /../../tests/system/providers/cohere/example_cohere_embedding_operator.py + :language: python + :dedent: 4 + :start-after: [START howto_cohere_operator] + :end-before: [END howto_cohere_operator] diff --git a/tests/system/providers/cohere/__init__.py b/tests/system/providers/cohere/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/tests/system/providers/cohere/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/tests/system/providers/cohere/example_cohere_embedding_operator.py b/tests/system/providers/cohere/example_cohere_embedding_operator.py new file mode 100644 index 0000000000000..97380965d0f3d --- /dev/null +++ b/tests/system/providers/cohere/example_cohere_embedding_operator.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +from datetime import datetime + +from airflow import DAG +from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator + +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") +DAG_ID = "example_cohere_embedding" +CONN_ID = "cohere_default" + +# [START howto_cohere_operator] +with DAG(DAG_ID, schedule=None, start_date=datetime(2023, 1, 1), catchup=False) as dag: + texts = [ + "On Kernel-Target Alignment. We describe a family of global optimization procedures", + " that automatically decompose optimization problems into smaller loosely coupled", + " problems, then combine the solutions of these with message passing algorithms.", + ] + + def get_text(): + return texts + + CohereEmbeddingOperator(input_text=texts, task_id="embedding_via_text", conn_id=CONN_ID) + CohereEmbeddingOperator(input_callable=get_text, task_id="embedding_via_callable", conn_id=CONN_ID) +# [START howto_cohere_operator] + + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag) From 84da2ebb90c55cb85cf5cd69e183ba3e79c5f330 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 17:05:41 +0530 Subject: [PATCH 07/36] Add dependency to of cohere python sdk --- docs/apache-airflow-providers-cohere/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index aab09cad3ca93..e72eb73bb7d24 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -91,6 +91,7 @@ The minimum Apache Airflow version supported by this provider package is ``2.4.0 PIP package Version required ================== ================== ``apache-airflow`` ``>=2.4.0`` +``cohere`` ``4.27`` ================== ================== .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! From e6810090e878b9b99da7a5ec924de9d0a01f493f Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 17:06:56 +0530 Subject: [PATCH 08/36] Add cache_property for the cohere client --- airflow/providers/cohere/hooks/cohere.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index c578f4124d126..30505ee8b358c 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +from functools import cached_property from typing import Any import cohere @@ -49,13 +50,12 @@ def get_conn(self) -> None: def _get_api_key(self) -> str: return str(self.get_connection(self.conn_id).password) - def get_client(self) -> cohere.Client: - if self.client is None: - self.client = cohere.Client(self._get_api_key()) - return self.client + @cached_property + def cohere_client(self) -> cohere.Client: + return cohere.Client(self._get_api_key()) def embed_text(self, texts: list[str], model: str = "embed-multilingual-v2.0") -> list[list[float]]: - response = self.get_client().embed(texts=texts, model=model) + response = self.cohere_client.embed(texts=texts, model=model) embeddings = response.embeddings return embeddings # type:ignore[no-any-return] @@ -68,7 +68,7 @@ def get_ui_field_behaviour() -> dict[str, Any]: def test_connection(self) -> tuple[bool, str]: try: - self.get_client().list_custom_models() + self.cohere_client.generate("Test", max_tokens=10) return True, "Connection established" except Exception as e: return False, str(e) From bbebd93fcbe2951d122a58f7f1b9ad4300e87777 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 17:12:17 +0530 Subject: [PATCH 09/36] Remove unwanted get_conn method --- airflow/providers/cohere/hooks/cohere.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index 30505ee8b358c..a1de61b89b4ae 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -44,9 +44,6 @@ def __init__(self, conn_id: str = default_conn_name) -> None: self.conn_id = conn_id self.client = None - def get_conn(self) -> None: - raise NotImplementedError() - def _get_api_key(self) -> str: return str(self.get_connection(self.conn_id).password) From 96d2f97ca8dc66c60ef9f4d7b4f0a14e006c2814 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 17:23:15 +0530 Subject: [PATCH 10/36] Add correct label to password field --- airflow/providers/cohere/hooks/cohere.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index a1de61b89b4ae..74f8b36049e07 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -60,7 +60,9 @@ def embed_text(self, texts: list[str], model: str = "embed-multilingual-v2.0") - def get_ui_field_behaviour() -> dict[str, Any]: return { "hidden_fields": ["host", "schema", "login", "port", "extra"], - "relabeling": {}, + "relabeling": { + "password": "API Key", + }, } def test_connection(self) -> tuple[bool, str]: From 90d469563fc116cebe7afdcd47ee9811341b33ca Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 16 Oct 2023 19:52:12 +0530 Subject: [PATCH 11/36] Expose timeout, max_retries and api_url to user --- airflow/providers/cohere/hooks/cohere.py | 19 +++++++++++++------ .../providers/cohere/operators/embedding.py | 8 +++++++- .../connections.rst | 3 +++ tests/providers/cohere/hooks/test_cohere.py | 17 +++++++++++++---- .../cohere/operators/test_embedding.py | 18 +++++++++++++----- 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index 74f8b36049e07..264ac04a3d42f 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -39,17 +39,24 @@ class CohereHook(BaseHook): conn_type = "cohere" hook_name = "Cohere" - def __init__(self, conn_id: str = default_conn_name) -> None: + def __init__( + self, + conn_id: str = default_conn_name, + timeout: int | None = None, + max_retries: int | None = None, + ) -> None: super().__init__() self.conn_id = conn_id self.client = None - - def _get_api_key(self) -> str: - return str(self.get_connection(self.conn_id).password) + self.timeout = timeout + self.max_retries = max_retries @cached_property def cohere_client(self) -> cohere.Client: - return cohere.Client(self._get_api_key()) + conn = self.get_connection(self.conn_id) + return cohere.Client( + api_key=conn.password, timeout=self.timeout, max_retries=self.max_retries, api_url=conn.host + ) def embed_text(self, texts: list[str], model: str = "embed-multilingual-v2.0") -> list[list[float]]: response = self.cohere_client.embed(texts=texts, model=model) @@ -59,7 +66,7 @@ def embed_text(self, texts: list[str], model: str = "embed-multilingual-v2.0") - @staticmethod def get_ui_field_behaviour() -> dict[str, Any]: return { - "hidden_fields": ["host", "schema", "login", "port", "extra"], + "hidden_fields": ["schema", "login", "port", "extra"], "relabeling": { "password": "API Key", }, diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index aede80441fdb7..de1aaaaa5deca 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -40,6 +40,8 @@ class CohereEmbeddingOperator(BaseOperator): Only one of input_text or input_callable should be provided. :param input_callable_args: The list of arguments to be passed to ``input_callable`` :param input_callable_kwargs: The kwargs to be passed to ``input_callable`` + :param timeout: Timeout in seconds for Cohere API + :param max_retries : No. of times to retry before failing. """ def __init__( @@ -49,6 +51,8 @@ def __init__( input_callable_args: Collection[Any] | None = None, input_callable_kwargs: Mapping[str, Any] | None = None, conn_id: str = CohereHook.default_conn_name, + timeout: int | None = None, + max_retries: int | None = None, **kwargs: Any, ): super().__init__(**kwargs) @@ -57,12 +61,14 @@ def __init__( self.input_callable = input_callable self.input_callable_args = input_callable_args or () self.input_callable_kwargs = input_callable_kwargs or {} + self.timeout = timeout + self.max_retries = max_retries self.text_to_embed = self._get_text_to_embed() @cached_property def hook(self) -> CohereHook: """Return an instance of the CohereHook.""" - return CohereHook(conn_id=self.conn_id) + return CohereHook(conn_id=self.conn_id, timeout=self.timeout, max_retries=self.max_retries) def _get_text_to_embed(self) -> list[str]: """Get the text to embed by evaluating ``input_text`` and ``input_callable``.""" diff --git a/docs/apache-airflow-providers-cohere/connections.rst b/docs/apache-airflow-providers-cohere/connections.rst index bfbc8b0ce0cf5..0ab583dfd59e0 100644 --- a/docs/apache-airflow-providers-cohere/connections.rst +++ b/docs/apache-airflow-providers-cohere/connections.rst @@ -32,3 +32,6 @@ Configuring the Connection Password (required) Specify the password to connect. + +Host (optional) + Specify the API host diff --git a/tests/providers/cohere/hooks/test_cohere.py b/tests/providers/cohere/hooks/test_cohere.py index 9d8dd4f088df9..dc5c8d4ddd1e0 100644 --- a/tests/providers/cohere/hooks/test_cohere.py +++ b/tests/providers/cohere/hooks/test_cohere.py @@ -31,8 +31,17 @@ class TestCohereHook: def test__get_api_key(self): api_key = "test" + api_url = "http://some_host.com" + timeout = 150 + max_retries = 5 with patch.object( - CohereHook, "get_connection", return_value=Connection(conn_type="cohere", password=api_key) - ): - hook = CohereHook() - assert api_key == hook._get_api_key() + CohereHook, + "get_connection", + return_value=Connection(conn_type="cohere", password=api_key, host=api_url), + ), patch("cohere.Client") as client: + + hook = CohereHook(timeout=timeout, max_retries=max_retries) + _ = hook.cohere_client + client.assert_called_once_with( + api_key=api_key, timeout=timeout, max_retries=max_retries, api_url=api_url + ) diff --git a/tests/providers/cohere/operators/test_embedding.py b/tests/providers/cohere/operators/test_embedding.py index 8703141b776a9..e22984778193d 100644 --- a/tests/providers/cohere/operators/test_embedding.py +++ b/tests/providers/cohere/operators/test_embedding.py @@ -21,12 +21,13 @@ import pytest from airflow.exceptions import AirflowException +from airflow.models import Connection from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator -@patch("airflow.providers.cohere.hooks.cohere.CohereHook._get_api_key") +@patch("airflow.providers.cohere.hooks.cohere.CohereHook.get_connection") @patch("cohere.Client") -def test_cohere_embedding_operator(cohere_client, _get_api_key): +def test_cohere_embedding_operator(cohere_client, get_connection): """ Test Cohere client is getting called with the correct key and that the execute methods returns expected response. @@ -37,17 +38,24 @@ class resp: embeddings = embedded_obj api_key = "test" + api_url = "http://some_host.com" + timeout = 150 + max_retries = 5 texts = ["On Kernel-Target Alignment. We describe a family of global optimization procedures"] - _get_api_key.return_value = api_key + get_connection.return_value = Connection(conn_type="cohere", password=api_key, host=api_url) client_obj = MagicMock() cohere_client.return_value = client_obj client_obj.embed.return_value = resp - op = CohereEmbeddingOperator(task_id="embed", conn_id="some_conn", input_text=texts) + op = CohereEmbeddingOperator( + task_id="embed", conn_id="some_conn", input_text=texts, timeout=timeout, max_retries=max_retries + ) val = op.execute(context={}) - cohere_client.assert_called_once_with(api_key) + cohere_client.assert_called_once_with( + api_key=api_key, api_url=api_url, timeout=timeout, max_retries=max_retries + ) assert val == embedded_obj From d60cfa012f2a40ac344485a2c7dc7c643b4e346f Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 25 Oct 2023 17:03:33 +0530 Subject: [PATCH 12/36] Fix documentation --- airflow/providers/cohere/hooks/cohere.py | 4 +++- airflow/providers/cohere/operators/embedding.py | 17 ++++++++++------- .../apache-airflow-providers-cohere/commits.rst | 3 +++ docs/apache-airflow-providers-cohere/index.rst | 11 +---------- .../installing-providers-from-sources.rst | 2 +- .../operators/{cohere.rst => embedding.rst} | 4 ---- .../cohere/example_cohere_embedding_operator.py | 2 +- 7 files changed, 19 insertions(+), 24 deletions(-) rename docs/apache-airflow-providers-cohere/operators/{cohere.rst => embedding.rst} (90%) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index 264ac04a3d42f..8253949c4cdbe 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -58,7 +58,9 @@ def cohere_client(self) -> cohere.Client: api_key=conn.password, timeout=self.timeout, max_retries=self.max_retries, api_url=conn.host ) - def embed_text(self, texts: list[str], model: str = "embed-multilingual-v2.0") -> list[list[float]]: + def create_embeddings( + self, texts: list[str], model: str = "embed-multilingual-v2.0" + ) -> list[list[float]]: response = self.cohere_client.embed(texts=texts, model=model) embeddings = response.embeddings return embeddings # type:ignore[no-any-return] diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index de1aaaaa5deca..4c903a6fbd4db 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -29,8 +29,11 @@ class CohereEmbeddingOperator(BaseOperator): - """ - Creates the embedding base by interacting with Cohere's hosted services. + """Creates the embedding base by interacting with cohere hosted services. + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:CohereEmbeddingOperator` :param conn_id: Optional. The name of the Airflow connection to get connection information for Cohere. Defaults to "cohere_default". @@ -38,10 +41,10 @@ class CohereEmbeddingOperator(BaseOperator): should be provided. :param input_callable: The callable that provides the input texts to generate embeddings for. Only one of input_text or input_callable should be provided. - :param input_callable_args: The list of arguments to be passed to ``input_callable`` - :param input_callable_kwargs: The kwargs to be passed to ``input_callable`` - :param timeout: Timeout in seconds for Cohere API - :param max_retries : No. of times to retry before failing. + :param input_callable_args: The list of arguments to be passed to ``input_callable``. + :param input_callable_kwargs: The kwargs to be passed to ``input_callable``. + :param timeout: Timeout in seconds for Cohere API. + :param max_retries: No. of times to retry before failing. """ def __init__( @@ -85,4 +88,4 @@ def _get_text_to_embed(self) -> list[str]: def execute(self, context: Context) -> list[list[float]]: """Embed texts using Cohere embed services.""" - return self.hook.embed_text(self.text_to_embed) + return self.hook.create_embeddings(self.text_to_embed) diff --git a/docs/apache-airflow-providers-cohere/commits.rst b/docs/apache-airflow-providers-cohere/commits.rst index 106592bd11775..f69ac2c0ad76c 100644 --- a/docs/apache-airflow-providers-cohere/commits.rst +++ b/docs/apache-airflow-providers-cohere/commits.rst @@ -14,3 +14,6 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +Package apache-airflow-providers-cohere +------------------------------------------- diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index e72eb73bb7d24..ef4338d817059 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -33,16 +33,7 @@ :caption: Guides Connection types - Operators - - -.. toctree:: - :hidden: - :maxdepth: 1 - :caption: Commits - - Detailed list of commits - + Operators .. toctree:: :hidden: diff --git a/docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst b/docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst index 1c90205d15b3a..b4e730f4ff21a 100644 --- a/docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst +++ b/docs/apache-airflow-providers-cohere/installing-providers-from-sources.rst @@ -15,4 +15,4 @@ specific language governing permissions and limitations under the License. -.. include:: ../installing-providers-from-sources.rst +.. include:: ../exts/includes/installing-providers-from-sources.rst diff --git a/docs/apache-airflow-providers-cohere/operators/cohere.rst b/docs/apache-airflow-providers-cohere/operators/embedding.rst similarity index 90% rename from docs/apache-airflow-providers-cohere/operators/cohere.rst rename to docs/apache-airflow-providers-cohere/operators/embedding.rst index 795e3f0329ec0..4b07e0025da5b 100644 --- a/docs/apache-airflow-providers-cohere/operators/cohere.rst +++ b/docs/apache-airflow-providers-cohere/operators/embedding.rst @@ -20,11 +20,7 @@ CohereEmbeddingOperator ======================== -<<<<<<< HEAD -Use the :class:`~astronomer_providers_llm.providers.cohere.operators.embedding.CohereEmbeddingOperator` to -======= Use the :class:`~airflow.providers.cohere.operators.embedding.CohereEmbeddingOperator` to ->>>>>>> d50e02cd6b (Add Cohere Provider) interact with Cohere APIs to create embeddings for a given text. diff --git a/tests/system/providers/cohere/example_cohere_embedding_operator.py b/tests/system/providers/cohere/example_cohere_embedding_operator.py index 97380965d0f3d..75e646e8f187e 100644 --- a/tests/system/providers/cohere/example_cohere_embedding_operator.py +++ b/tests/system/providers/cohere/example_cohere_embedding_operator.py @@ -39,7 +39,7 @@ def get_text(): CohereEmbeddingOperator(input_text=texts, task_id="embedding_via_text", conn_id=CONN_ID) CohereEmbeddingOperator(input_callable=get_text, task_id="embedding_via_callable", conn_id=CONN_ID) -# [START howto_cohere_operator] +# [END howto_cohere_operator] from tests.system.utils import get_test_run # noqa: E402 From aeb37afe1fcc1b8595818af32dfeaddee54c337b Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 25 Oct 2023 19:55:22 +0530 Subject: [PATCH 13/36] Update interface of CohereEmbeddingOperator operator --- INSTALL | 1 - .../providers/cohere/operators/embedding.py | 29 ++++--------------- airflow/providers/cohere/provider.yaml | 2 +- .../apache-airflow-providers-cohere/index.rst | 2 +- generated/provider_dependencies.json | 2 +- .../example_cohere_embedding_operator.py | 25 ++++++++-------- 6 files changed, 21 insertions(+), 40 deletions(-) diff --git a/INSTALL b/INSTALL index f060a49324b8f..c45cce8e42688 100644 --- a/INSTALL +++ b/INSTALL @@ -108,7 +108,6 @@ oracle, otel, pagerduty, pandas, papermill, password, pinot, plexus, postgres, p redis, s3, s3fs, salesforce, samba, segment, sendgrid, sentry, sftp, singularity, slack, smtp, snowflake, spark, sqlite, ssh, statsd, tableau, tabular, telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk - # END EXTRAS HERE # For installing Airflow in development environments - see CONTRIBUTING.rst diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index 4c903a6fbd4db..37fb2f664861a 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -18,9 +18,8 @@ from __future__ import annotations from functools import cached_property -from typing import TYPE_CHECKING, Any, Callable, Collection, Mapping +from typing import TYPE_CHECKING, Any, Sequence -from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.cohere.hooks.cohere import CohereHook @@ -47,12 +46,11 @@ class CohereEmbeddingOperator(BaseOperator): :param max_retries: No. of times to retry before failing. """ + template_fields: Sequence[str] = ("input_text",) + def __init__( self, - input_text: list[str] | None = None, - input_callable: Callable[[Any], list[str]] | None = None, - input_callable_args: Collection[Any] | None = None, - input_callable_kwargs: Mapping[str, Any] | None = None, + input_text: list[str], conn_id: str = CohereHook.default_conn_name, timeout: int | None = None, max_retries: int | None = None, @@ -61,31 +59,14 @@ def __init__( super().__init__(**kwargs) self.conn_id = conn_id self.input_text = input_text - self.input_callable = input_callable - self.input_callable_args = input_callable_args or () - self.input_callable_kwargs = input_callable_kwargs or {} self.timeout = timeout self.max_retries = max_retries - self.text_to_embed = self._get_text_to_embed() @cached_property def hook(self) -> CohereHook: """Return an instance of the CohereHook.""" return CohereHook(conn_id=self.conn_id, timeout=self.timeout, max_retries=self.max_retries) - def _get_text_to_embed(self) -> list[str]: - """Get the text to embed by evaluating ``input_text`` and ``input_callable``.""" - if self.input_text and self.input_callable: - raise RuntimeError("Only one of 'input_text' and 'input_callable' is allowed") - if self.input_callable: - if not callable(self.input_callable): - raise AirflowException("`input_callable` param must be callable") - return self.input_callable(*self.input_callable_args, **self.input_callable_kwargs) - elif self.input_text: - return self.input_text - else: - raise RuntimeError("Either one of 'input_text' and 'input_callable' must be provided") - def execute(self, context: Context) -> list[list[float]]: """Embed texts using Cohere embed services.""" - return self.hook.create_embeddings(self.text_to_embed) + return self.hook.create_embeddings(self.input_text) diff --git a/airflow/providers/cohere/provider.yaml b/airflow/providers/cohere/provider.yaml index 37427709c9360..83c3bf4f1ed9f 100644 --- a/airflow/providers/cohere/provider.yaml +++ b/airflow/providers/cohere/provider.yaml @@ -35,7 +35,7 @@ integrations: dependencies: - apache-airflow>=2.5.0 - - cohere + - cohere>=4.27 hooks: - integration-name: Cohere diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index ef4338d817059..3a368e70cf491 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -82,7 +82,7 @@ The minimum Apache Airflow version supported by this provider package is ``2.4.0 PIP package Version required ================== ================== ``apache-airflow`` ``>=2.4.0`` -``cohere`` ``4.27`` +``cohere`` ``>=4.27`` ================== ================== .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 2d58848370f9a..8be76a03e9d9e 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -272,7 +272,7 @@ "cohere": { "deps": [ "apache-airflow>=2.5.0", - "cohere" + "cohere>=4.27" ], "cross-providers-deps": [], "excluded-python-versions": [] diff --git a/tests/system/providers/cohere/example_cohere_embedding_operator.py b/tests/system/providers/cohere/example_cohere_embedding_operator.py index 75e646e8f187e..47cbcd364790d 100644 --- a/tests/system/providers/cohere/example_cohere_embedding_operator.py +++ b/tests/system/providers/cohere/example_cohere_embedding_operator.py @@ -16,33 +16,34 @@ # under the License. from __future__ import annotations -import os from datetime import datetime from airflow import DAG +from airflow.decorators import task from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator -ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") -DAG_ID = "example_cohere_embedding" -CONN_ID = "cohere_default" - -# [START howto_cohere_operator] -with DAG(DAG_ID, schedule=None, start_date=datetime(2023, 1, 1), catchup=False) as dag: +with DAG("example_cohere_embedding", schedule=None, start_date=datetime(2023, 1, 1), catchup=False) as dag: + # [START howto_operator_cohere_embedding] texts = [ "On Kernel-Target Alignment. We describe a family of global optimization procedures", " that automatically decompose optimization problems into smaller loosely coupled", " problems, then combine the solutions of these with message passing algorithms.", ] - def get_text(): + @task + def task_to_get_text(): + return texts + + def get_task(): return texts - CohereEmbeddingOperator(input_text=texts, task_id="embedding_via_text", conn_id=CONN_ID) - CohereEmbeddingOperator(input_callable=get_text, task_id="embedding_via_callable", conn_id=CONN_ID) -# [END howto_cohere_operator] + CohereEmbeddingOperator(input_text=texts, task_id="embedding_via_text") + CohereEmbeddingOperator(input_text=task_to_get_text(), task_id="embedding_via_task") + CohereEmbeddingOperator(input_text=get_task(), task_id="embedding_via_callable") + # [END howto_operator_cohere_embedding] -from tests.system.utils import get_test_run # noqa: E402 +from tests.system.utils import get_test_run # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) test_run = get_test_run(dag) From e38fdda2725b32f5cde222541d75a46da9a00cdc Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 25 Oct 2023 20:02:21 +0530 Subject: [PATCH 14/36] Updated testcases --- docs/apache-airflow-providers-cohere/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index 3a368e70cf491..ffd0cc0fef077 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -76,12 +76,12 @@ for the minimum Airflow version supported) via Requirements ------------ -The minimum Apache Airflow version supported by this provider package is ``2.4.0``. +The minimum Apache Airflow version supported by this provider package is ``2.5.0``. ================== ================== PIP package Version required ================== ================== -``apache-airflow`` ``>=2.4.0`` +``apache-airflow`` ``>=2.5.0`` ``cohere`` ``>=4.27`` ================== ================== From 649ead599d8b8136acb5df23e119956ec26fcf0b Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 25 Oct 2023 20:03:10 +0530 Subject: [PATCH 15/36] Updated testcases --- .../cohere/operators/test_embedding.py | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/tests/providers/cohere/operators/test_embedding.py b/tests/providers/cohere/operators/test_embedding.py index e22984778193d..32dd83aa2614a 100644 --- a/tests/providers/cohere/operators/test_embedding.py +++ b/tests/providers/cohere/operators/test_embedding.py @@ -18,9 +18,6 @@ from unittest.mock import MagicMock, patch -import pytest - -from airflow.exceptions import AirflowException from airflow.models import Connection from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator @@ -57,38 +54,3 @@ class resp: api_key=api_key, api_url=api_url, timeout=timeout, max_retries=max_retries ) assert val == embedded_obj - - -@pytest.mark.parametrize( - "inputs, expected_exception, error_message", - ( - ( - {"input_text": "test", "input_callable": "some_value"}, - RuntimeError, - "Only one of 'input_text' and 'input_callable' is allowed", - ), - ({"input_callable": "str_type"}, AirflowException, "`input_callable` param must be callable"), - ({}, RuntimeError, "Either one of 'input_text' and 'input_callable' must be provided"), - ), -) -def test_validate_inputs(inputs, expected_exception, error_message): - print( - "inputs : ", inputs, " expected_exception : ", expected_exception, " error_message: ", error_message - ) - with pytest.raises(expected_exception, match=error_message): - CohereEmbeddingOperator(task_id="embed", conn_id="some_conn", **inputs) - - -@pytest.mark.parametrize( - "inputs, expected_text", - ( - ({"input_text": ["some text", "some text"]}, ["some text", "some text"]), - ( - {"input_callable": lambda: ["some other text", "some other text"]}, - ["some other text", "some other text"], - ), - ), -) -def test_get_text(inputs, expected_text): - op = CohereEmbeddingOperator(task_id="embed", conn_id="some_conn", **inputs) - assert op.text_to_embed == expected_text From 4dec45c596d4675e2fc35ef29cd061b4ac70840b Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Thu, 26 Oct 2023 15:32:44 +0530 Subject: [PATCH 16/36] Fix static check and docs build --- airflow/providers/cohere/provider.yaml | 4 +++- docs/apache-airflow-providers-cohere/operators/embedding.rst | 4 ++-- tests/providers/cohere/hooks/test_cohere.py | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airflow/providers/cohere/provider.yaml b/airflow/providers/cohere/provider.yaml index 83c3bf4f1ed9f..55500e0e760ff 100644 --- a/airflow/providers/cohere/provider.yaml +++ b/airflow/providers/cohere/provider.yaml @@ -31,6 +31,8 @@ versions: integrations: - integration-name: Cohere external-doc-url: https://docs.cohere.com/docs + how-to-guide: + - /docs/apache-airflow-providers-cohere/operators/embedding.rst tags: [software] dependencies: @@ -43,7 +45,7 @@ hooks: - airflow.providers.cohere.hooks.cohere operators: - - integration-name: Cohere Embedding Model + - integration-name: Cohere python-modules: - airflow.providers.cohere.operators.embedding diff --git a/docs/apache-airflow-providers-cohere/operators/embedding.rst b/docs/apache-airflow-providers-cohere/operators/embedding.rst index 4b07e0025da5b..a8f198116355e 100644 --- a/docs/apache-airflow-providers-cohere/operators/embedding.rst +++ b/docs/apache-airflow-providers-cohere/operators/embedding.rst @@ -38,5 +38,5 @@ Example Code: .. exampleinclude:: /../../tests/system/providers/cohere/example_cohere_embedding_operator.py :language: python :dedent: 4 - :start-after: [START howto_cohere_operator] - :end-before: [END howto_cohere_operator] + :start-after: [START howto_operator_cohere_embedding] + :end-before: [END howto_operator_cohere_embedding] diff --git a/tests/providers/cohere/hooks/test_cohere.py b/tests/providers/cohere/hooks/test_cohere.py index dc5c8d4ddd1e0..9af272222bfa9 100644 --- a/tests/providers/cohere/hooks/test_cohere.py +++ b/tests/providers/cohere/hooks/test_cohere.py @@ -39,7 +39,6 @@ def test__get_api_key(self): "get_connection", return_value=Connection(conn_type="cohere", password=api_key, host=api_url), ), patch("cohere.Client") as client: - hook = CohereHook(timeout=timeout, max_retries=max_retries) _ = hook.cohere_client client.assert_called_once_with( From 9341c177861d56f064382ce5b4539edded59fde8 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Tue, 31 Oct 2023 19:35:47 +0530 Subject: [PATCH 17/36] Update CONTRIBUTING.rst Co-authored-by: Josh Fell <48934154+josh-fell@users.noreply.github.com> --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 7abf7ad2b9dab..74dba76481946 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1444,7 +1444,7 @@ You can join the channels via links at the `Airflow Community page `_ for: * detailed discussions on big proposals (Airflow Improvement Proposals also name AIPs) * helpful, shared resources (for example Apache Airflow logos - * information that can be re-used by others (for example instructions on preparing workshops) + * information that can be reused by others (for example instructions on preparing workshops) * GitHub `Pull Requests (PRs) `_ for: * discussing implementation details of PRs * not for architectural discussions (use the devlist for that) From 22aad729f21c64d826dd21c97b5ec8ec3168dc8b Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Tue, 31 Oct 2023 19:37:05 +0530 Subject: [PATCH 18/36] Update docs/apache-airflow-providers-cohere/operators/embedding.rst Co-authored-by: Josh Fell <48934154+josh-fell@users.noreply.github.com> --- docs/apache-airflow-providers-cohere/operators/embedding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-cohere/operators/embedding.rst b/docs/apache-airflow-providers-cohere/operators/embedding.rst index a8f198116355e..1aa105106509c 100644 --- a/docs/apache-airflow-providers-cohere/operators/embedding.rst +++ b/docs/apache-airflow-providers-cohere/operators/embedding.rst @@ -27,7 +27,7 @@ interact with Cohere APIs to create embeddings for a given text. Using the Operator ^^^^^^^^^^^^^^^^^^ -The CohereEmbeddingOperator requires the ``texts`` as an input to embedding API. Use the ``cohere_conn_id`` parameter to specify the Cohere connection to use to +The CohereEmbeddingOperator requires the ``input_text`` as an input to embedding API. Use the ``cohere_conn_id`` parameter to specify the Cohere connection to use to connect to your account. An example using the operator is in way: From c9444d3a2864c5f864d369a6f34a6dd55ce88deb Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Tue, 31 Oct 2023 19:37:46 +0530 Subject: [PATCH 19/36] Update airflow/providers/cohere/operators/embedding.py Co-authored-by: Josh Fell <48934154+josh-fell@users.noreply.github.com> --- airflow/providers/cohere/operators/embedding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index 37fb2f664861a..592c2dffdcbe0 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -43,7 +43,7 @@ class CohereEmbeddingOperator(BaseOperator): :param input_callable_args: The list of arguments to be passed to ``input_callable``. :param input_callable_kwargs: The kwargs to be passed to ``input_callable``. :param timeout: Timeout in seconds for Cohere API. - :param max_retries: No. of times to retry before failing. + :param max_retries: Number of times to retry before failing. """ template_fields: Sequence[str] = ("input_text",) From b8d1b6e0fd39524723bfc5c9a9198656f87124a9 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Wed, 1 Nov 2023 15:13:35 +0530 Subject: [PATCH 20/36] Update airflow/providers/cohere/hooks/cohere.py Co-authored-by: Hussein Awala --- airflow/providers/cohere/hooks/cohere.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index 8253949c4cdbe..3188cdce0ab2b 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -52,7 +52,7 @@ def __init__( self.max_retries = max_retries @cached_property - def cohere_client(self) -> cohere.Client: + def get_conn(self) -> cohere.Client: conn = self.get_connection(self.conn_id) return cohere.Client( api_key=conn.password, timeout=self.timeout, max_retries=self.max_retries, api_url=conn.host From 914c631ed3d97592f0788b562a425aeeebdccc71 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Wed, 1 Nov 2023 15:14:40 +0530 Subject: [PATCH 21/36] Update airflow/providers/cohere/CHANGELOG.rst Co-authored-by: Hussein Awala --- airflow/providers/cohere/CHANGELOG.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/airflow/providers/cohere/CHANGELOG.rst b/airflow/providers/cohere/CHANGELOG.rst index 450e8ed420cd7..d199887ba773e 100644 --- a/airflow/providers/cohere/CHANGELOG.rst +++ b/airflow/providers/cohere/CHANGELOG.rst @@ -15,8 +15,10 @@ specific language governing permissions and limitations under the License. +``apache-airflow-providers-cohere`` -.. NOTE TO CONTRIBUTORS: - Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes - and you want to add an explanation to the users on how they are supposed to deal with them. - The changelog is updated and maintained semi-automatically by release manager. +Changelog +--------- + +1.0.0 +..... From 07316a80fbf70534214c92ad05051b44b7743c7e Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 1 Nov 2023 16:53:08 +0530 Subject: [PATCH 22/36] Address the PR comments --- airflow/providers/cohere/hooks/cohere.py | 4 +++- airflow/providers/cohere/operators/embedding.py | 8 +++----- .../operators/embedding.rst | 2 -- .../cohere/example_cohere_embedding_operator.py | 11 +---------- 4 files changed, 7 insertions(+), 18 deletions(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index 3188cdce0ab2b..f58d8b1686d82 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -32,6 +32,8 @@ class CohereHook(BaseHook): .. seealso:: https://docs.cohere.com/docs :param conn_id: :ref:`Cohere connection id ` + :param timeout: Request timeout in seconds. + :param max_retries: Maximal number of retries for requests. """ conn_name_attr = "conn_id" @@ -63,7 +65,7 @@ def create_embeddings( ) -> list[list[float]]: response = self.cohere_client.embed(texts=texts, model=model) embeddings = response.embeddings - return embeddings # type:ignore[no-any-return] + return embeddings @staticmethod def get_ui_field_behaviour() -> dict[str, Any]: diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index 592c2dffdcbe0..6385286daf2a0 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -38,10 +38,6 @@ class CohereEmbeddingOperator(BaseOperator): information for Cohere. Defaults to "cohere_default". :param input_text: list of text items that need to be embedded. Only one of input_text or input_callable should be provided. - :param input_callable: The callable that provides the input texts to generate embeddings for. - Only one of input_text or input_callable should be provided. - :param input_callable_args: The list of arguments to be passed to ``input_callable``. - :param input_callable_kwargs: The kwargs to be passed to ``input_callable``. :param timeout: Timeout in seconds for Cohere API. :param max_retries: Number of times to retry before failing. """ @@ -50,13 +46,15 @@ class CohereEmbeddingOperator(BaseOperator): def __init__( self, - input_text: list[str], + input_text: list[str] | str, conn_id: str = CohereHook.default_conn_name, timeout: int | None = None, max_retries: int | None = None, **kwargs: Any, ): super().__init__(**kwargs) + if isinstance(input_text, str): + input_text = [input_text] self.conn_id = conn_id self.input_text = input_text self.timeout = timeout diff --git a/docs/apache-airflow-providers-cohere/operators/embedding.rst b/docs/apache-airflow-providers-cohere/operators/embedding.rst index 1aa105106509c..577449c1ace75 100644 --- a/docs/apache-airflow-providers-cohere/operators/embedding.rst +++ b/docs/apache-airflow-providers-cohere/operators/embedding.rst @@ -30,8 +30,6 @@ Using the Operator The CohereEmbeddingOperator requires the ``input_text`` as an input to embedding API. Use the ``cohere_conn_id`` parameter to specify the Cohere connection to use to connect to your account. -An example using the operator is in way: - Example Code: ------------- diff --git a/tests/system/providers/cohere/example_cohere_embedding_operator.py b/tests/system/providers/cohere/example_cohere_embedding_operator.py index 47cbcd364790d..ec97ee91e57cb 100644 --- a/tests/system/providers/cohere/example_cohere_embedding_operator.py +++ b/tests/system/providers/cohere/example_cohere_embedding_operator.py @@ -19,7 +19,6 @@ from datetime import datetime from airflow import DAG -from airflow.decorators import task from airflow.providers.cohere.operators.embedding import CohereEmbeddingOperator with DAG("example_cohere_embedding", schedule=None, start_date=datetime(2023, 1, 1), catchup=False) as dag: @@ -30,16 +29,8 @@ " problems, then combine the solutions of these with message passing algorithms.", ] - @task - def task_to_get_text(): - return texts - - def get_task(): - return texts - CohereEmbeddingOperator(input_text=texts, task_id="embedding_via_text") - CohereEmbeddingOperator(input_text=task_to_get_text(), task_id="embedding_via_task") - CohereEmbeddingOperator(input_text=get_task(), task_id="embedding_via_callable") + CohereEmbeddingOperator(input_text=texts[0], task_id="embedding_via_task") # [END howto_operator_cohere_embedding] From 2254142527a6d0a0029357e45bc0ea4fcff1ea63 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Wed, 1 Nov 2023 18:26:25 +0530 Subject: [PATCH 23/36] Resolve conflicts --- images/breeze/output_build-docs.svg | 2 +- ...ut_release-management_add-back-references.svg | 8 ++++---- ...nagement_generate-issue-content-providers.svg | 12 ++++++------ ...management_prepare-provider-documentation.svg | 12 ++++++------ ...ease-management_prepare-provider-packages.svg | 12 ++++++------ .../output_release-management_publish-docs.svg | 2 +- ...tput_sbom_generate-providers-requirements.svg | 16 ++++++++-------- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg index 6bb4ceffe65a2..7622be26f3eb2 100644 --- a/images/breeze/output_build-docs.svg +++ b/images/breeze/output_build-docs.svg @@ -163,7 +163,7 @@ [OPTIONS] [all-providers | providers-index | apache-airflow | docker-stack | helm-chart | airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive |              apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop |  -apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | common.io | common.sql |           +apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql |  daskexecutor | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |    ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie |   diff --git a/images/breeze/output_release-management_add-back-references.svg b/images/breeze/output_release-management_add-back-references.svg index ebcd1572ef1b6..fd0739906cc3b 100644 --- a/images/breeze/output_release-management_add-back-references.svg +++ b/images/breeze/output_release-management_add-back-references.svg @@ -134,10 +134,10 @@ [OPTIONS] [all-providers | apache-airflow | docker-stack | helm-chart | airbyte | alibaba | amazon | apache.beam |     apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.impala |            apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise |        -arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | common.io | common.sql | daskexecutor |      -databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook | ftp | github |    -google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql |              -microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie |     +arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql |            +daskexecutor | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |    +ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql +| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie |   oracle | pagerduty | papermill | plexus | postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex |      zendesk]...                                                                                                            diff --git a/images/breeze/output_release-management_generate-issue-content-providers.svg b/images/breeze/output_release-management_generate-issue-content-providers.svg index 10139cafc51e5..6f03ffa40db55 100644 --- a/images/breeze/output_release-management_generate-issue-content-providers.svg +++ b/images/breeze/output_release-management_generate-issue-content-providers.svg @@ -144,12 +144,12 @@ [OPTIONS] [apache-airflow | docker-stack | helm-chart | airbyte | alibaba | amazon | apache.beam | apache.cassandra |  apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira |  -celery | cloudant | cncf.kubernetes | common.io | common.sql | daskexecutor | databricks | datadog | dbt.cloud |       -dingding | discord | docker | elasticsearch | exasol | facebook | ftp | github | google | grpc | hashicorp | http |    -imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo |      -mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | plexus |      -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | zendesk]...                                   +celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql | daskexecutor | databricks | datadog |          +dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook | ftp | github | google | grpc | hashicorp +| http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm |     +mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill |       +plexus | postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | zendesk]...                       Generates content for issue to test the release. diff --git a/images/breeze/output_release-management_prepare-provider-documentation.svg b/images/breeze/output_release-management_prepare-provider-documentation.svg index a8e4587c4b15c..27f198d97e31d 100644 --- a/images/breeze/output_release-management_prepare-provider-documentation.svg +++ b/images/breeze/output_release-management_prepare-provider-documentation.svg @@ -156,12 +156,12 @@ [OPTIONS] [apache-airflow | docker-stack | helm-chart | airbyte | alibaba | amazon | apache.beam | apache.cassandra |  apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira |  -celery | cloudant | cncf.kubernetes | common.io | common.sql | daskexecutor | databricks | datadog | dbt.cloud |       -dingding | discord | docker | elasticsearch | exasol | facebook | ftp | github | google | grpc | hashicorp | http |    -imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo |      -mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | plexus |      -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | zendesk]...                                   +celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql | daskexecutor | databricks | datadog |          +dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook | ftp | github | google | grpc | hashicorp +| http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm |     +mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill |       +plexus | postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | zendesk]...                       Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/images/breeze/output_release-management_prepare-provider-packages.svg b/images/breeze/output_release-management_prepare-provider-packages.svg index fa4c36063edd9..507cd13783471 100644 --- a/images/breeze/output_release-management_prepare-provider-packages.svg +++ b/images/breeze/output_release-management_prepare-provider-packages.svg @@ -141,12 +141,12 @@ [OPTIONS] [apache-airflow | docker-stack | helm-chart | airbyte | alibaba | amazon | apache.beam | apache.cassandra |  apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb | asana | atlassian.jira |  -celery | cloudant | cncf.kubernetes | common.io | common.sql | daskexecutor | databricks | datadog | dbt.cloud |       -dingding | discord | docker | elasticsearch | exasol | facebook | ftp | github | google | grpc | hashicorp | http |    -imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo |      -mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | plexus |      -postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |  -sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | zendesk]...                                   +celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql | daskexecutor | databricks | datadog |          +dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook | ftp | github | google | grpc | hashicorp +| http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm |     +mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill |       +plexus | postgres | presto | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp |     +snowflake | sqlite | ssh | tableau | tabular | telegram | trino | vertica | yandex | zendesk]...                       Prepare sdist/whl packages of Airflow Providers. diff --git a/images/breeze/output_release-management_publish-docs.svg b/images/breeze/output_release-management_publish-docs.svg index 8108ee78d10a7..38ffcaf32d39f 100644 --- a/images/breeze/output_release-management_publish-docs.svg +++ b/images/breeze/output_release-management_publish-docs.svg @@ -180,7 +180,7 @@ [OPTIONS] [all-providers | providers-index | apache-airflow | docker-stack | helm-chart | airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive |              apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop |  -apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | common.io | common.sql |           +apprise | arangodb | asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql |  daskexecutor | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch | exasol | facebook |    ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie |   diff --git a/images/breeze/output_sbom_generate-providers-requirements.svg b/images/breeze/output_sbom_generate-providers-requirements.svg index 9f08035158704..6dfe112fdf2eb 100644 --- a/images/breeze/output_sbom_generate-providers-requirements.svg +++ b/images/breeze/output_sbom_generate-providers-requirements.svg @@ -186,14 +186,14 @@ │(airbyte | alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid |  │ │apache.flink | apache.hdfs | apache.hive | apache.impala | apache.kafka | apache.kylin |      │ │apache.livy | apache.pig | apache.pinot | apache.spark | apache.sqoop | apprise | arangodb |  │ -│asana | atlassian.jira | celery | cloudant | cncf.kubernetes | common.io | common.sql |       │ -│daskexecutor | databricks | datadog | dbt.cloud | dingding | discord | docker | elasticsearch │ -│| exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap | influxdb | jdbc│ -│| jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo |    │ -│mysql | neo4j | odbc | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty |  │ -│papermill | plexus | postgres | presto | redis | salesforce | samba | segment | sendgrid |    │ -│sftp | singularity | slack | smtp | snowflake | sqlite | ssh | tableau | tabular | telegram | │ -│trino | vertica | yandex | zendesk)                                                           │ +│asana | atlassian.jira | celery | cloudant | cncf.kubernetes | cohere | common.io | common.sql│ +│| daskexecutor | databricks | datadog | dbt.cloud | dingding | discord | docker |             │ +│elasticsearch | exasol | facebook | ftp | github | google | grpc | hashicorp | http | imap |  │ +│influxdb | jdbc | jenkins | microsoft.azure | microsoft.mssql | microsoft.psrp |              │ +│microsoft.winrm | mongo | mysql | neo4j | odbc | openfaas | openlineage | opensearch |        │ +│opsgenie | oracle | pagerduty | papermill | plexus | postgres | presto | redis | salesforce | │ +│samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh |   │ +│tableau | tabular | telegram | trino | vertica | yandex | zendesk)                            │ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported   │ │value to account for the most recent version of the provider                                  │ │(TEXT)                                                                                        │ From d479161759258481b8767b7ec18e717ab8d06433 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Thu, 2 Nov 2023 18:02:35 +0530 Subject: [PATCH 24/36] Fix breaking tests --- airflow/providers/cohere/hooks/cohere.py | 4 ++-- tests/providers/cohere/hooks/test_cohere.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index f58d8b1686d82..1c5005c0c4a82 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -63,7 +63,7 @@ def get_conn(self) -> cohere.Client: def create_embeddings( self, texts: list[str], model: str = "embed-multilingual-v2.0" ) -> list[list[float]]: - response = self.cohere_client.embed(texts=texts, model=model) + response = self.get_conn.embed(texts=texts, model=model) embeddings = response.embeddings return embeddings @@ -78,7 +78,7 @@ def get_ui_field_behaviour() -> dict[str, Any]: def test_connection(self) -> tuple[bool, str]: try: - self.cohere_client.generate("Test", max_tokens=10) + self.get_conn.generate("Test", max_tokens=10) return True, "Connection established" except Exception as e: return False, str(e) diff --git a/tests/providers/cohere/hooks/test_cohere.py b/tests/providers/cohere/hooks/test_cohere.py index 9af272222bfa9..8f566ec0c6405 100644 --- a/tests/providers/cohere/hooks/test_cohere.py +++ b/tests/providers/cohere/hooks/test_cohere.py @@ -40,7 +40,7 @@ def test__get_api_key(self): return_value=Connection(conn_type="cohere", password=api_key, host=api_url), ), patch("cohere.Client") as client: hook = CohereHook(timeout=timeout, max_retries=max_retries) - _ = hook.cohere_client + _ = hook.get_conn client.assert_called_once_with( api_key=api_key, timeout=timeout, max_retries=max_retries, api_url=api_url ) From 40ed8eea8f1fff136c1ca17edfdaffdc218f3886 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Thu, 2 Nov 2023 18:24:11 +0530 Subject: [PATCH 25/36] Fix static checks --- CONTRIBUTING.rst | 6 +++--- INSTALL | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 74dba76481946..15783d1c9cd91 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -671,9 +671,9 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks, datadog, -dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, -doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, +cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks, +datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, +doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, openlineage, opensearch, opsgenie, diff --git a/INSTALL b/INSTALL index c45cce8e42688..6e233fe0a0556 100644 --- a/INSTALL +++ b/INSTALL @@ -98,9 +98,9 @@ aiobotocore, airbyte, alibaba, all, all_dbs, amazon, apache.atlas, apache.beam, apache.drill, apache.druid, apache.flink, apache.hdfs, apache.hive, apache.impala, apache.kafka, apache.kylin, apache.livy, apache.pig, apache.pinot, apache.spark, apache.sqoop, apache.webhdfs, apprise, arangodb, asana, async, atlas, atlassian.jira, aws, azure, cassandra, celery, cgroups, -cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks, datadog, -dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, doc, -doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, +cloudant, cncf.kubernetes, cohere, common.io, common.sql, crypto, dask, daskexecutor, databricks, +datadog, dbt.cloud, deprecated_api, devel, devel_all, devel_ci, devel_hadoop, dingding, discord, +doc, doc_gen, docker, druid, elasticsearch, exasol, facebook, ftp, gcp, gcp_api, github, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, influxdb, jdbc, jenkins, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql, microsoft.psrp, microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, openlineage, opensearch, opsgenie, From 38d79ffc0668ef212c1bf37be7b1342c96b63a05 Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Thu, 2 Nov 2023 19:32:21 +0530 Subject: [PATCH 26/36] Update airflow/providers/cohere/operators/embedding.py Co-authored-by: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> --- airflow/providers/cohere/operators/embedding.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index 6385286daf2a0..c001712fc5ba4 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -34,10 +34,10 @@ class CohereEmbeddingOperator(BaseOperator): For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:CohereEmbeddingOperator` + :param input_text: list of text items that need to be embedded. Only one of input_text or input_callable + should be provided. :param conn_id: Optional. The name of the Airflow connection to get connection information for Cohere. Defaults to "cohere_default". - :param input_text: list of text items that need to be embedded. Only one of input_text or input_callable - should be provided. :param timeout: Timeout in seconds for Cohere API. :param max_retries: Number of times to retry before failing. """ From 120c4c1a3bc4ae29164fab07fb98fc709f07f191 Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Fri, 3 Nov 2023 15:12:17 +0530 Subject: [PATCH 27/36] Fix docstring --- airflow/providers/cohere/operators/embedding.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index c001712fc5ba4..776f2c27ddeed 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -34,8 +34,7 @@ class CohereEmbeddingOperator(BaseOperator): For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:CohereEmbeddingOperator` - :param input_text: list of text items that need to be embedded. Only one of input_text or input_callable - should be provided. + :param input_text: list of text items that need to be embedded. :param conn_id: Optional. The name of the Airflow connection to get connection information for Cohere. Defaults to "cohere_default". :param timeout: Timeout in seconds for Cohere API. From 6460b10b6a1f135e821d059665fd8ce4ee3b141e Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Fri, 3 Nov 2023 15:15:20 +0530 Subject: [PATCH 28/36] Add note for initial release --- airflow/providers/cohere/CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airflow/providers/cohere/CHANGELOG.rst b/airflow/providers/cohere/CHANGELOG.rst index d199887ba773e..54baa1945c135 100644 --- a/airflow/providers/cohere/CHANGELOG.rst +++ b/airflow/providers/cohere/CHANGELOG.rst @@ -22,3 +22,5 @@ Changelog 1.0.0 ..... + +Initial version of the provider. From c590d4fd3a39665a2f24a9f656f209295e1db98e Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Fri, 3 Nov 2023 20:25:50 +0530 Subject: [PATCH 29/36] Add security.rst file --- .../security.rst | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/apache-airflow-providers-cohere/security.rst diff --git a/docs/apache-airflow-providers-cohere/security.rst b/docs/apache-airflow-providers-cohere/security.rst new file mode 100644 index 0000000000000..66c6f79a4ecfc --- /dev/null +++ b/docs/apache-airflow-providers-cohere/security.rst @@ -0,0 +1,38 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Releasing security patches +-------------------------- + +Airflow providers are released independently from Airflow itself and the information about vulnerabilities +is published separately. You can upgrade providers independently from Airflow itself, following the +instructions found in :doc:`apache-airflow:installation/installing-from-pypi`. + +When we release Provider version, the development is always done from the ``main`` branch where we prepare +the next version. The provider uses strict `SemVer `_ versioning policy. Depending on +the scope of the change, Provider will get ''MAJOR'' version upgrade when there are +breaking changes, ``MINOR`` version upgrade when there are new features or ``PATCHLEVEL`` version upgrade +when there are only bug fixes (including security bugfixes) - and this is the only version that receives +security fixes by default, so you should upgrade to latest version of the provider if you want to receive +all released security fixes. + +The only exception to that rule is when we have a critical security fix and good reason to provide an +out-of-band release for the provider, in which case stakeholders in the provider might decide to cherry-pick +and prepare a branch for an older version of the provider following the +`mixed governance model `_ +and requires interested parties to cherry-pick and test the fixes. From 02898c775e990ea3f9dc7e8da827c06057e223fe Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Sat, 4 Nov 2023 03:31:52 +0530 Subject: [PATCH 30/36] Update airflow/providers/cohere/hooks/cohere.py --- airflow/providers/cohere/hooks/cohere.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/cohere/hooks/cohere.py b/airflow/providers/cohere/hooks/cohere.py index 1c5005c0c4a82..20c77d58690e9 100644 --- a/airflow/providers/cohere/hooks/cohere.py +++ b/airflow/providers/cohere/hooks/cohere.py @@ -49,7 +49,6 @@ def __init__( ) -> None: super().__init__() self.conn_id = conn_id - self.client = None self.timeout = timeout self.max_retries = max_retries From 9d8aca388d3706e84f3714a7da0e13a93da0728c Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Sat, 4 Nov 2023 03:32:04 +0530 Subject: [PATCH 31/36] Update airflow/providers/cohere/operators/embedding.py Co-authored-by: Josh Fell <48934154+josh-fell@users.noreply.github.com> --- airflow/providers/cohere/operators/embedding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/cohere/operators/embedding.py b/airflow/providers/cohere/operators/embedding.py index 776f2c27ddeed..dba95e7e8f661 100644 --- a/airflow/providers/cohere/operators/embedding.py +++ b/airflow/providers/cohere/operators/embedding.py @@ -34,7 +34,7 @@ class CohereEmbeddingOperator(BaseOperator): For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:CohereEmbeddingOperator` - :param input_text: list of text items that need to be embedded. + :param input_text: single string text or list of text items that need to be embedded. :param conn_id: Optional. The name of the Airflow connection to get connection information for Cohere. Defaults to "cohere_default". :param timeout: Timeout in seconds for Cohere API. From bcf94c1d3aa40a0687ba7a9b75b2785837882bcd Mon Sep 17 00:00:00 2001 From: Utkarsh Sharma Date: Sat, 4 Nov 2023 03:34:59 +0530 Subject: [PATCH 32/36] Update docs/apache-airflow-providers-cohere/operators/embedding.rst Co-authored-by: Josh Fell <48934154+josh-fell@users.noreply.github.com> --- docs/apache-airflow-providers-cohere/operators/embedding.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/apache-airflow-providers-cohere/operators/embedding.rst b/docs/apache-airflow-providers-cohere/operators/embedding.rst index 577449c1ace75..b765fe8e9d07c 100644 --- a/docs/apache-airflow-providers-cohere/operators/embedding.rst +++ b/docs/apache-airflow-providers-cohere/operators/embedding.rst @@ -27,7 +27,7 @@ interact with Cohere APIs to create embeddings for a given text. Using the Operator ^^^^^^^^^^^^^^^^^^ -The CohereEmbeddingOperator requires the ``input_text`` as an input to embedding API. Use the ``cohere_conn_id`` parameter to specify the Cohere connection to use to +The CohereEmbeddingOperator requires the ``input_text`` as an input to embedding API. Use the ``conn_id`` parameter to specify the Cohere connection to use to connect to your account. Example Code: From 2f54f72d3952695051edacb178fac1d5ed649226 Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Sat, 4 Nov 2023 17:38:00 +0530 Subject: [PATCH 33/36] Add ref to sequrity.rst --- docs/apache-airflow-providers-cohere/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index ffd0cc0fef077..e9d1883b48a2f 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -26,6 +26,7 @@ :caption: Basics Home + Security .. toctree:: :hidden: From f21a85d8ad7ad9cfeeb86dace9076b6fc3459527 Mon Sep 17 00:00:00 2001 From: Pankaj Singh <98807258+pankajastro@users.noreply.github.com> Date: Sat, 4 Nov 2023 20:18:05 +0530 Subject: [PATCH 34/36] Update docs/apache-airflow-providers-cohere/security.rst --- .../security.rst | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/docs/apache-airflow-providers-cohere/security.rst b/docs/apache-airflow-providers-cohere/security.rst index 66c6f79a4ecfc..b8f95e6ecfa29 100644 --- a/docs/apache-airflow-providers-cohere/security.rst +++ b/docs/apache-airflow-providers-cohere/security.rst @@ -16,23 +16,4 @@ specific language governing permissions and limitations under the License. -Releasing security patches --------------------------- - -Airflow providers are released independently from Airflow itself and the information about vulnerabilities -is published separately. You can upgrade providers independently from Airflow itself, following the -instructions found in :doc:`apache-airflow:installation/installing-from-pypi`. - -When we release Provider version, the development is always done from the ``main`` branch where we prepare -the next version. The provider uses strict `SemVer `_ versioning policy. Depending on -the scope of the change, Provider will get ''MAJOR'' version upgrade when there are -breaking changes, ``MINOR`` version upgrade when there are new features or ``PATCHLEVEL`` version upgrade -when there are only bug fixes (including security bugfixes) - and this is the only version that receives -security fixes by default, so you should upgrade to latest version of the provider if you want to receive -all released security fixes. - -The only exception to that rule is when we have a critical security fix and good reason to provide an -out-of-band release for the provider, in which case stakeholders in the provider might decide to cherry-pick -and prepare a branch for an older version of the provider following the -`mixed governance model `_ -and requires interested parties to cherry-pick and test the fixes. +.. include:: ../exts/includes/security.rst From c730abb211ae68280745c0b9a5e5025924c1a10b Mon Sep 17 00:00:00 2001 From: Pankaj Date: Sat, 4 Nov 2023 21:13:46 +0530 Subject: [PATCH 35/36] Add /changelog.rst --- .../changelog.rst | 18 ++++++++++++++++++ docs/apache-airflow-providers-cohere/index.rst | 1 + 2 files changed, 19 insertions(+) create mode 100644 docs/apache-airflow-providers-cohere/changelog.rst diff --git a/docs/apache-airflow-providers-cohere/changelog.rst b/docs/apache-airflow-providers-cohere/changelog.rst new file mode 100644 index 0000000000000..be95cf4557aa1 --- /dev/null +++ b/docs/apache-airflow-providers-cohere/changelog.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../airflow/providers/cohere/CHANGELOG.rst diff --git a/docs/apache-airflow-providers-cohere/index.rst b/docs/apache-airflow-providers-cohere/index.rst index e9d1883b48a2f..8bfd1678a3460 100644 --- a/docs/apache-airflow-providers-cohere/index.rst +++ b/docs/apache-airflow-providers-cohere/index.rst @@ -26,6 +26,7 @@ :caption: Basics Home + Changelog Security .. toctree:: From 7f1e0ae6216a0281ca59dd702deab611e4f6fe1b Mon Sep 17 00:00:00 2001 From: utkarsh sharma Date: Mon, 6 Nov 2023 15:59:31 +0530 Subject: [PATCH 36/36] Resolve conflicts --- images/breeze/output-commands-hash.txt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 64ba323baa4ad..a5d9a89c8d328 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,7 +2,7 @@ # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. main:96b4884054753db922cb8ca2cc555368 -build-docs:2e9882744f219e56726548ce2d13c3f5 +build-docs:3863424b58f31d5a2b285055b044ad70 ci:find-backtracking-candidates:17fe56b867a745e5032a08dfcd3f73ee ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96 ci:free-space:49af17b032039c05c41a7a8283f365cc @@ -36,26 +36,26 @@ prod-image:build:1628f7bff3e7e369f0358a646682e674 prod-image:pull:3817ef211b023b76df84ee1110ef64dd prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 prod-image:6011405076eb0e1049d87e971e3adce1 -release-management:add-back-references:51960e2831d0e03a2b127d252929b843 +release-management:add-back-references:8154089f6b96923edd772c5a38f7d143 release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d release-management:generate-constraints:01aef235b11e59ed7f10c970a5cdaba7 -release-management:generate-issue-content-providers:cda108e7f2506c2816af8f2a6c24070c +release-management:generate-issue-content-providers:97e29b10f93a0d0276469fc470110c95 release-management:generate-providers-metadata:d4e8e5cfaa024e3963af02d7a873048d release-management:install-provider-packages:34c38aca17d23dbb454fe7a6bfd8e630 release-management:prepare-airflow-package:85d01c57e5b5ee0fb9e5f9d9706ed3b5 -release-management:prepare-provider-documentation:eb861d68b8d72cd98dc8732fc5393796 -release-management:prepare-provider-packages:908e2c826f7b4959dfd8bc693f3857a7 -release-management:publish-docs:51ee9bf1268529513996a14bd5350c19 +release-management:prepare-provider-documentation:8a142fef4a148279d7794571c4cfed33 +release-management:prepare-provider-packages:10c9e86ce1ba6c50acfd0d5367d06da8 +release-management:publish-docs:7eef8e54cbc9743afb865531ff92503d release-management:release-prod-images:cfbfe8b19fee91fd90718f98ef2fd078 release-management:start-rc-process:b27bd524dd3c89f50a747b60a7e892c1 release-management:start-release:419f48f6a4ff4457cb9de7ff496aebbe release-management:update-constraints:02ec4b119150e3fdbac52026e94820ef release-management:verify-provider-packages:96dce5644aad6b37080acf77b3d8de3a -release-management:59d956e45fccf55e47f16e33cfc5d04a +release-management:c0d470f72e53df330e573c7bc0a08470 sbom:build-all-airflow-images:32f8acade299c2b112e986bae99846db -sbom:generate-providers-requirements:3926848718283cf2ef00310a0892e867 +sbom:generate-providers-requirements:bc008fcd52f258fbd989a42e66e44349 sbom:update-sbom-information:653be48be70b4b7ff5172d491aadc694 -sbom:386048e0c00c0de30cf181eb9f3862ea +sbom:9d0e848adefac54a4ccda950ca3bc4ee setup:autocomplete:fffcd49e102e09ccd69b3841a9e3ea8e setup:check-all-params-in-groups:5c5e3c382fc8ce84899d224448b3f48a setup:config:3435f1f1535a82c30591dbf577294d2e