Skip to content

Deprecate AutoML Tables operators #39752

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eladkal merged 2 commits into from
May 24, 2024
Merged

Deprecate AutoML Tables operators #39752

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
982d092
Deprecate AutoML Table operators
e-galan May 20, 2024
a4eb18e
Update providers.google.CHANGELOG.rst with info about AutoML shutdown
e-galan May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions airflow/providers/google/CHANGELOG.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,21 @@
Changelog
---------

Main
.......

.. note::
Several AutoML operators have stopped being supported following the shutdown of a legacy version of
AutoML Natural Language, Tables, Vision, and Video Intelligence services. This includes
``AutoMLDeployModelOperator``, ``AutoMLTablesUpdateDatasetOperator``, ``AutoMLTablesListTableSpecsOperator``
and ``AutoMLTablesListColumnSpecsOperator``. Please refer to the operator documentation to find out
about available alternatives, if any. For additional information regarding the AutoML shutdown see:

* `AutoML Natural Language <https://cloud.google.com/natural-language/automl/docs/deprecations>`_
* `AutoML Tables <https://cloud.google.com/automl-tables/docs/deprecations>`_
* `AutoML Vision <https://cloud.google.com/vision/automl/docs/deprecations>`_
* `AutoML Video Intelligence <https://cloud.google.com/video-intelligence/automl/docs/deprecations>`_

10.18.0
.......

Expand Down
73 changes: 40 additions & 33 deletions airflow/providers/google/cloud/operators/automl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from functools import cached_property
from typing import TYPE_CHECKING, Sequence, Tuple

from deprecated import deprecated
from google.api_core.gapic_v1.method import DEFAULT, _MethodDefault
from google.cloud.automl_v1beta1 import (
BatchPredictResult,
Expand All @@ -35,7 +34,7 @@
TableSpec,
)

from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
from airflow.exceptions import AirflowException
from airflow.providers.google.cloud.hooks.automl import CloudAutoMLHook
from airflow.providers.google.cloud.hooks.vertex_ai.prediction_service import PredictionServiceHook
from airflow.providers.google.cloud.links.translate import (
Expand Down Expand Up @@ -685,6 +684,10 @@ class AutoMLTablesListColumnSpecsOperator(GoogleCloudBaseOperator):
"""
Lists column specs in a table.

Operator AutoMLTablesListColumnSpecsOperator has been deprecated due to shutdown of
a legacy version of AutoML Tables on March 31, 2024. For additional information
see: https://cloud.google.com/automl-tables/docs/deprecations.

.. seealso::
For more information on how to use this operator, take a look at the guide:
:ref:`howto/operator:AutoMLTablesListColumnSpecsOperator`
Expand Down Expand Up @@ -759,6 +762,11 @@ def __init__(
self.retry = retry
self.gcp_conn_id = gcp_conn_id
self.impersonation_chain = impersonation_chain
raise AirflowException(
"Operator AutoMLTablesListColumnSpecsOperator has been deprecated due to shutdown of "
"a legacy version of AutoML Tables on March 31, 2024. "
"For additional information see: https://cloud.google.com/automl-tables/docs/deprecations."
)

def execute(self, context: Context):
hook = CloudAutoMLHook(
Expand Down Expand Up @@ -791,20 +799,14 @@ def execute(self, context: Context):
return result


@deprecated(
reason=(
"Class `AutoMLTablesUpdateDatasetOperator` has been deprecated and no longer available. "
"Please use `UpdateDatasetOperator` instead"
),
category=AirflowProviderDeprecationWarning,
action="error",
)
class AutoMLTablesUpdateDatasetOperator(GoogleCloudBaseOperator):
"""
Updates a dataset.

AutoMLTablesUpdateDatasetOperator has been deprecated and no longer available. Please use
:class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.UpdateDatasetOperator`
Operator AutoMLTablesUpdateDatasetOperator has been deprecated due to shutdown of
a legacy version of AutoML Tables on March 31, 2024. For additional information
see: https://cloud.google.com/automl-tables/docs/deprecations.
Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.dataset.UpdateDatasetOperator`
instead.

.. seealso::
Expand Down Expand Up @@ -864,6 +866,12 @@ def __init__(
self.retry = retry
self.gcp_conn_id = gcp_conn_id
self.impersonation_chain = impersonation_chain
raise AirflowException(
"Operator AutoMLTablesUpdateDatasetOperator has been deprecated due to shutdown of "
"a legacy version of AutoML Tables on March 31, 2024. "
"For additional information see: https://cloud.google.com/automl-tables/docs/deprecations. "
"Please use UpdateDatasetOperator from Vertex AI instead."
)

def execute(self, context: Context):
hook = CloudAutoMLHook(
Expand Down Expand Up @@ -1074,14 +1082,6 @@ def execute(self, context: Context):
self.log.info("Deletion is completed")


@deprecated(
reason=(
"Class `AutoMLDeployModelOperator` has been deprecated and no longer available. Please use "
"`DeployModelOperator` instead"
),
category=AirflowProviderDeprecationWarning,
action="error",
)
class AutoMLDeployModelOperator(GoogleCloudBaseOperator):
"""
Deploys a model; if a model is already deployed, deploying it with the same parameters has no effect.
Expand All @@ -1092,8 +1092,10 @@ class AutoMLDeployModelOperator(GoogleCloudBaseOperator):
Only applicable for Text Classification, Image Object Detection and Tables; all other
domains manage deployment automatically.

AutoMLDeployModelOperator has been deprecated and no longer available. Please use
:class:`airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.DeployModelOperator`
Operator AutoMLDeployModelOperator has been deprecated due to shutdown of a legacy version
of AutoML Natural Language, Vision, Video Intelligence on March 31, 2024.
For additional information see: https://cloud.google.com/vision/automl/docs/deprecations .
Please use :class:`airflow.providers.google.cloud.operators.vertex_ai.endpoint_service.DeployModelOperator`
instead.

.. seealso::
Expand Down Expand Up @@ -1156,24 +1158,20 @@ def __init__(
self.retry = retry
self.gcp_conn_id = gcp_conn_id
self.impersonation_chain = impersonation_chain
raise AirflowException(
"Operator AutoMLDeployModelOperator has been deprecated due to shutdown of "
"a legacy version of AutoML AutoML Natural Language, Vision, Video Intelligence "
"on March 31, 2024. "
"For additional information see: https://cloud.google.com/vision/automl/docs/deprecations. "
"Please use DeployModelOperator from Vertex AI instead."
)

def execute(self, context: Context):
hook = CloudAutoMLHook(
gcp_conn_id=self.gcp_conn_id,
impersonation_chain=self.impersonation_chain,
)
model = hook.get_model(
model_id=self.model_id,
location=self.location,
project_id=self.project_id,
retry=self.retry,
timeout=self.timeout,
metadata=self.metadata,
)
if not hasattr(model, "translation_model_metadata"):
_raise_exception_for_deprecated_operator(self.__class__.__name__, "DeployModelOperator")
self.log.info("Deploying model_id %s", self.model_id)

operation = hook.deploy_model(
model_id=self.model_id,
location=self.location,
Expand All @@ -1191,6 +1189,10 @@ class AutoMLTablesListTableSpecsOperator(GoogleCloudBaseOperator):
"""
Lists table specs in a dataset.

Operator AutoMLTablesListTableSpecsOperator has been deprecated due to shutdown of
a legacy version of AutoML Tables on March 31, 2024. For additional information
see: https://cloud.google.com/automl-tables/docs/deprecations.

.. seealso::
For more information on how to use this operator, take a look at the guide:
:ref:`howto/operator:AutoMLTablesListTableSpecsOperator`
Expand Down Expand Up @@ -1256,6 +1258,11 @@ def __init__(
self.retry = retry
self.gcp_conn_id = gcp_conn_id
self.impersonation_chain = impersonation_chain
raise AirflowException(
"Operator AutoMLTablesListTableSpecsOperator has been deprecated due to shutdown of "
"a legacy version of AutoML Tables on March 31, 2024. "
"For additional information see: https://cloud.google.com/automl-tables/docs/deprecations. "
)

def execute(self, context: Context):
hook = CloudAutoMLHook(
Expand Down
2 changes: 2 additions & 0 deletions scripts/in_container/run_provider_yaml_files_check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@

KNOWN_DEPRECATED_CLASSES = [
"airflow.providers.google.cloud.links.dataproc.DataprocLink",
"airflow.providers.google.cloud.operators.automl.AutoMLTablesListColumnSpecsOperator",
"airflow.providers.google.cloud.operators.automl.AutoMLTablesListTableSpecsOperator",
"airflow.providers.google.cloud.operators.automl.AutoMLTablesUpdateDatasetOperator",
"airflow.providers.google.cloud.operators.automl.AutoMLDeployModelOperator",
]
Expand Down
2 changes: 2 additions & 0 deletions tests/always/test_project_structure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,8 @@ class TestGoogleProviderProjectStructure(ExampleCoverageTest, AssetsCoverageTest
".CloudDataTransferServiceS3ToGCSOperator",
"airflow.providers.google.cloud.operators.cloud_storage_transfer_service"
".CloudDataTransferServiceGCSToGCSOperator",
"airflow.providers.google.cloud.operators.automl.AutoMLTablesListColumnSpecsOperator",
"airflow.providers.google.cloud.operators.automl.AutoMLTablesListTableSpecsOperator",
"airflow.providers.google.cloud.operators.automl.AutoMLTablesUpdateDatasetOperator",
"airflow.providers.google.cloud.operators.automl.AutoMLDeployModelOperator",
"airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHadoopJobOperator",
Expand Down
Loading