diff --git a/sdks/python/apache_beam/io/gcp/pubsublite/__init__.py b/sdks/python/apache_beam/io/gcp/pubsublite/__init__.py deleted file mode 100644 index 565777e14050..000000000000 --- a/sdks/python/apache_beam/io/gcp/pubsublite/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .proto_api import ReadFromPubSubLite -from .proto_api import WriteToPubSubLite - -__all__ = [ - "ReadFromPubSubLite", - "WriteToPubSubLite", -] diff --git a/sdks/python/apache_beam/io/gcp/pubsublite/external.py b/sdks/python/apache_beam/io/gcp/pubsublite/external.py deleted file mode 100644 index a0e46c1b4d88..000000000000 --- a/sdks/python/apache_beam/io/gcp/pubsublite/external.py +++ /dev/null @@ -1,115 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Google Pub/Sub Lite sources and sinks. - -This API is currently under development and is subject to change. -""" - -# pytype: skip-file - -import typing - -from apache_beam.transforms.external import BeamJarExpansionService -from apache_beam.transforms.external import ExternalTransform -from apache_beam.transforms.external import NamedTupleBasedPayloadBuilder - -_ReadSchema = typing.NamedTuple( - '_ReadSchema', [('subscription_path', str), ('deduplicate', bool)]) - - -def _default_io_expansion_service(): - return BeamJarExpansionService( - 'sdks:java:io:google-cloud-platform:expansion-service:shadowJar') - - -class _ReadExternal(ExternalTransform): - """ - An external PTransform which reads from Pub/Sub Lite and returns a - SequencedMessage as serialized bytes. - - This transform is not part of the public API. - - Experimental; no backwards-compatibility guarantees. - """ - def __init__( - self, - subscription_path, - deduplicate=None, - expansion_service=None, - ): - """ - Initializes a read operation from Pub/Sub Lite, returning the serialized - bytes of SequencedMessage protos. - - Args: - subscription_path: A Pub/Sub Lite Subscription path. - deduplicate: Whether to deduplicate messages based on the value of - the 'x-goog-pubsublite-dataflow-uuid' attribute. - """ - if deduplicate is None: - deduplicate = False - if expansion_service is None: - expansion_service = _default_io_expansion_service() - super().__init__( - 'beam:transform:org.apache.beam:pubsublite_read:v1', - NamedTupleBasedPayloadBuilder( - _ReadSchema( - subscription_path=subscription_path, deduplicate=deduplicate)), - expansion_service) - - -_WriteSchema = typing.NamedTuple( - '_WriteSchema', [('topic_path', str), ('add_uuids', bool)]) - - -class _WriteExternal(ExternalTransform): - """ - An external PTransform which writes serialized PubSubMessage protos to - Pub/Sub Lite. - - This transform is not part of the public API. - - Experimental; no backwards-compatibility guarantees. - """ - def __init__( - self, - topic_path, - add_uuids=None, - expansion_service=None, - ): - """ - Initializes a write operation to Pub/Sub Lite, writing the serialized bytes - of PubSubMessage protos. - - Args: - topic_path: A Pub/Sub Lite Topic path. - add_uuids: Whether to add uuids to the 'x-goog-pubsublite-dataflow-uuid' - uuid attribute. - """ - if add_uuids is None: - add_uuids = False - if expansion_service is None: - expansion_service = _default_io_expansion_service() - super().__init__( - 'beam:transform:org.apache.beam:pubsublite_write:v1', - NamedTupleBasedPayloadBuilder( - _WriteSchema( - topic_path=topic_path, - add_uuids=add_uuids, - )), - expansion_service) diff --git a/sdks/python/apache_beam/io/gcp/pubsublite/proto_api.py b/sdks/python/apache_beam/io/gcp/pubsublite/proto_api.py deleted file mode 100644 index a8e3defc4f99..000000000000 --- a/sdks/python/apache_beam/io/gcp/pubsublite/proto_api.py +++ /dev/null @@ -1,106 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from apache_beam.io.gcp.pubsublite.external import _ReadExternal -from apache_beam.io.gcp.pubsublite.external import _WriteExternal -from apache_beam.transforms import Map -from apache_beam.transforms import PTransform - -try: - from google.cloud import pubsublite -except ImportError: - pubsublite = None - - -class ReadFromPubSubLite(PTransform): - """ - A ``PTransform`` for reading from Pub/Sub Lite. - - Produces a PCollection of google.cloud.pubsublite.SequencedMessage - - Experimental; no backwards-compatibility guarantees. - """ - def __init__( - self, - subscription_path, - deduplicate=None, - expansion_service=None, - ): - """Initializes ``ReadFromPubSubLite``. - - Args: - subscription_path: Pub/Sub Lite Subscription in the form - projects//locations//subscriptions/ - deduplicate: Whether to deduplicate messages based on the value of - the 'x-goog-pubsublite-dataflow-uuid' attribute. Defaults to False. - """ - super().__init__() - self._source = _ReadExternal( - subscription_path=subscription_path, - deduplicate=deduplicate, - expansion_service=expansion_service, - ) - - def expand(self, pvalue): - pcoll = pvalue.pipeline | self._source - pcoll.element_type = bytes - pcoll = pcoll | Map(pubsublite.SequencedMessage.deserialize) - pcoll.element_type = pubsublite.SequencedMessage - return pcoll - - -class WriteToPubSubLite(PTransform): - """ - A ``PTransform`` for writing to Pub/Sub Lite. - - Consumes a PCollection of google.cloud.pubsublite.PubSubMessage - - Experimental; no backwards-compatibility guarantees. - """ - def __init__( - self, - topic_path, - add_uuids=None, - expansion_service=None, - ): - """Initializes ``WriteToPubSubLite``. - - Args: - topic_path: A Pub/Sub Lite Topic path. - add_uuids: Whether to add uuids to the 'x-goog-pubsublite-dataflow-uuid' - uuid attribute. Defaults to False. - """ - super().__init__() - self._source = _WriteExternal( - topic_path=topic_path, - add_uuids=add_uuids, - expansion_service=expansion_service, - ) - - @staticmethod - def _message_to_proto_str(element: pubsublite.PubSubMessage): - if not isinstance(element, pubsublite.PubSubMessage): - raise TypeError( - 'Unexpected element. Type: %s (expected: PubSubMessage), ' - 'value: %r' % (type(element), element)) - return pubsublite.PubSubMessage.serialize(element) - - def expand(self, pcoll): - pcoll = pcoll | Map(WriteToPubSubLite._message_to_proto_str) - pcoll.element_type = bytes - pcoll = pcoll | self._source - return pcoll diff --git a/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/EmojiMap.ts b/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/EmojiMap.ts index ed6a9f2285c8..800ec6aa9eea 100644 --- a/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/EmojiMap.ts +++ b/sdks/python/apache_beam/runners/interactive/extensions/apache-beam-jupyterlab-sidepanel/src/yaml/EmojiMap.ts @@ -62,8 +62,6 @@ export const transformEmojiMap: Record = { WriteToPostgres: '⬆️🐘', ReadFromPubSub: '⬇️📢', WriteToPubSub: '⬆️📢', - ReadFromPubSubLite: '⬇️📣', - WriteToPubSubLite: '⬆️📣', ReadFromSpanner: '⬇️📏', WriteToSpanner: '⬆️📏', ReadFromSqlServer: '⬇️🗄️', diff --git a/sdks/python/container/ml/py310/base_image_requirements.txt b/sdks/python/container/ml/py310/base_image_requirements.txt index 729ab317e648..673e63f0ac95 100644 --- a/sdks/python/container/ml/py310/base_image_requirements.txt +++ b/sdks/python/container/ml/py310/base_image_requirements.txt @@ -82,7 +82,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/ml/py310/gpu_image_requirements.txt b/sdks/python/container/ml/py310/gpu_image_requirements.txt index ad5095bec5eb..553de91f9ef3 100644 --- a/sdks/python/container/ml/py310/gpu_image_requirements.txt +++ b/sdks/python/container/ml/py310/gpu_image_requirements.txt @@ -99,7 +99,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/ml/py311/base_image_requirements.txt b/sdks/python/container/ml/py311/base_image_requirements.txt index f078095f6948..cfdf71ce77bd 100644 --- a/sdks/python/container/ml/py311/base_image_requirements.txt +++ b/sdks/python/container/ml/py311/base_image_requirements.txt @@ -80,7 +80,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/ml/py311/gpu_image_requirements.txt b/sdks/python/container/ml/py311/gpu_image_requirements.txt index 1fd5d768406f..313cad9147bd 100644 --- a/sdks/python/container/ml/py311/gpu_image_requirements.txt +++ b/sdks/python/container/ml/py311/gpu_image_requirements.txt @@ -97,7 +97,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/ml/py312/base_image_requirements.txt b/sdks/python/container/ml/py312/base_image_requirements.txt index 0bbe666b6805..da889c0526d0 100644 --- a/sdks/python/container/ml/py312/base_image_requirements.txt +++ b/sdks/python/container/ml/py312/base_image_requirements.txt @@ -79,7 +79,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/ml/py312/gpu_image_requirements.txt b/sdks/python/container/ml/py312/gpu_image_requirements.txt index d0024977e5c0..ca40f87da0b2 100644 --- a/sdks/python/container/ml/py312/gpu_image_requirements.txt +++ b/sdks/python/container/ml/py312/gpu_image_requirements.txt @@ -96,7 +96,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/ml/py313/base_image_requirements.txt b/sdks/python/container/ml/py313/base_image_requirements.txt index 1a2e88342d24..5e3b32e82bd4 100644 --- a/sdks/python/container/ml/py313/base_image_requirements.txt +++ b/sdks/python/container/ml/py313/base_image_requirements.txt @@ -77,7 +77,6 @@ google-cloud-kms==3.9.0 google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/py310/base_image_requirements.txt b/sdks/python/container/py310/base_image_requirements.txt index cc61bef566fe..61c827e09630 100644 --- a/sdks/python/container/py310/base_image_requirements.txt +++ b/sdks/python/container/py310/base_image_requirements.txt @@ -76,7 +76,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/py311/base_image_requirements.txt b/sdks/python/container/py311/base_image_requirements.txt index 09e6e9dc1453..7be8ec580698 100644 --- a/sdks/python/container/py311/base_image_requirements.txt +++ b/sdks/python/container/py311/base_image_requirements.txt @@ -74,7 +74,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/py312/base_image_requirements.txt b/sdks/python/container/py312/base_image_requirements.txt index 398a4b8285c7..f2c9ce2786f2 100644 --- a/sdks/python/container/py312/base_image_requirements.txt +++ b/sdks/python/container/py312/base_image_requirements.txt @@ -73,7 +73,6 @@ google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-profiler==4.1.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/container/py313/base_image_requirements.txt b/sdks/python/container/py313/base_image_requirements.txt index d5b9331a9be2..6a919da4bc93 100644 --- a/sdks/python/container/py313/base_image_requirements.txt +++ b/sdks/python/container/py313/base_image_requirements.txt @@ -71,7 +71,6 @@ google-cloud-kms==3.9.0 google-cloud-language==2.19.0 google-cloud-monitoring==2.29.0 google-cloud-pubsub==2.34.0 -google-cloud-pubsublite==1.13.0 google-cloud-recommendations-ai==0.10.18 google-cloud-resource-manager==1.16.0 google-cloud-secret-manager==2.26.0 diff --git a/sdks/python/setup.py b/sdks/python/setup.py index bcdd1d77584a..0e27d99deb3b 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -479,7 +479,6 @@ def get_portability_package_data(): 'google-auth-httplib2>=0.1.0,<0.3.0', 'google-cloud-datastore>=2.0.0,<3', 'google-cloud-pubsub>=2.1.0,<3', - 'google-cloud-pubsublite>=1.2.0,<2', 'google-cloud-storage>=2.18.2,<3', # GCP packages required by tests 'google-cloud-bigquery>=2.0.0,<4',