diff --git a/docs/source/developers/python.rst b/docs/source/developers/python.rst index 188486cae37..fc48b2d65ec 100644 --- a/docs/source/developers/python.rst +++ b/docs/source/developers/python.rst @@ -109,7 +109,8 @@ The test groups currently include: * ``large_memory``: Test requiring a large amount of system RAM * ``orc``: Apache ORC tests * ``parquet``: Apache Parquet tests -* ``plasma``: Plasma Object Store tests +* ``plasma``: Plasma Object Store tests (deprecated since Arrow 10.0.0, + will be removed in 12.0.0 or so) * ``s3``: Tests for Amazon S3 * ``tensorflow``: Tests that involve TensorFlow @@ -330,7 +331,8 @@ adding flags with ``ON``: * ``ARROW_ORC``: Support for Apache ORC file format * ``ARROW_PARQUET``: Support for Apache Parquet file format * ``PARQUET_REQUIRE_ENCRYPTION``: Support for Parquet Modular Encryption -* ``ARROW_PLASMA``: Shared memory object store +* ``ARROW_PLASMA``: Shared memory object store (deprecated since Arrow 10.0.0, + will be removed in 12.0.0 or so) Anything set to ``ON`` above can also be turned off. Note that some compression libraries are recommended for full Parquet support. diff --git a/docs/source/python/api/plasma.rst b/docs/source/python/api/plasma.rst index 8df9e4e21ac..0ef21116cf4 100644 --- a/docs/source/python/api/plasma.rst +++ b/docs/source/python/api/plasma.rst @@ -22,6 +22,10 @@ Plasma In-Memory Object Store ============================= +.. warning:: + + Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so. + Classes ------- diff --git a/docs/source/python/plasma.rst b/docs/source/python/plasma.rst index 51c7b6eafee..c12a0ddbe96 100644 --- a/docs/source/python/plasma.rst +++ b/docs/source/python/plasma.rst @@ -21,6 +21,10 @@ The Plasma In-Memory Object Store ================================= +.. warning:: + + Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so. + .. note:: As present, Plasma is only supported for use on Linux and macOS. diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 19d630bc0ab..bcb6b30e715 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -341,13 +341,21 @@ def __getattr__(name): def _plasma_store_entry_point(): - """Entry point for starting the plasma store. + """ + DEPRECATED: Entry point for starting the plasma store. This can be used by invoking e.g. ``plasma_store -s /tmp/plasma -m 1000000000`` from the command line and will start the plasma_store executable with the given arguments. + + .. deprecated:: 10.0.0 + Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so. """ + warnings.warn( + "Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so.", + DeprecationWarning) + import pyarrow plasma_store_executable = _os.path.join(pyarrow.__path__[0], "plasma-store-server") diff --git a/python/pyarrow/_plasma.pyx b/python/pyarrow/_plasma.pyx index 35d39073634..61a4ac14651 100644 --- a/python/pyarrow/_plasma.pyx +++ b/python/pyarrow/_plasma.pyx @@ -156,7 +156,10 @@ def make_object_id(object_id): cdef class ObjectID(_Weakrefable): """ - An ObjectID represents a string of bytes used to identify Plasma objects. + DEPRECATED: An ObjectID represents a string of bytes used to identify Plasma objects. + + .. deprecated:: 10.0.0 + Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so. """ cdef: @@ -169,6 +172,10 @@ cdef class ObjectID(_Weakrefable): " is " + str(object_id)) self.data = CUniqueID.from_binary(object_id) + warnings.warn( + "Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so.", + DeprecationWarning, stacklevel=2) + def __eq__(self, other): try: return self.data == (other).data @@ -219,13 +226,16 @@ cdef class ObjectNotAvailable(_Weakrefable): cdef class PlasmaBuffer(Buffer): """ - This is the type returned by calls to get with a PlasmaClient. + DEPRECATED: This is the type returned by calls to get with a PlasmaClient. We define our own class instead of directly returning a buffer object so that we can add a custom destructor which notifies Plasma that the object is no longer being used, so the memory in the Plasma store backing the object can potentially be freed. + .. deprecated:: 10.0.0 + Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so. + Attributes ---------- object_id : ObjectID @@ -295,11 +305,14 @@ def get_socket_from_fd(fileno, family, type): cdef class PlasmaClient(_Weakrefable): """ - The PlasmaClient is used to interface with a plasma store and manager. + DEPRECATED: The PlasmaClient is used to interface with a plasma store and manager. The PlasmaClient can ask the PlasmaStore to allocate a new buffer, seal a buffer, and get a buffer. Buffers are referred to by object IDs, which are strings. + + .. deprecated:: 10.0.0 + Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so. """ cdef: @@ -312,6 +325,10 @@ cdef class PlasmaClient(_Weakrefable): self.notification_fd = -1 self.store_socket_name = b"" + warnings.warn( + "Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so.", + DeprecationWarning, stacklevel=3) + cdef _get_object_buffers(self, object_ids, int64_t timeout_ms, c_vector[CObjectBuffer]* result): cdef: @@ -854,9 +871,12 @@ cdef class PlasmaClient(_Weakrefable): def connect(store_socket_name, int num_retries=-1): """ - Return a new PlasmaClient that is connected a plasma store and + DEPRECATED: Return a new PlasmaClient that is connected a plasma store and optionally a manager. + .. deprecated:: 10.0.0 + Plasma is deprecated since Arrow 10.0.0. It will be removed in 12.0.0 or so. + Parameters ---------- store_socket_name : str diff --git a/python/pyarrow/plasma.py b/python/pyarrow/plasma.py index 712ed9e5b22..5c2c6543418 100644 --- a/python/pyarrow/plasma.py +++ b/python/pyarrow/plasma.py @@ -24,6 +24,7 @@ import sys import tempfile import time +import warnings from pyarrow._plasma import (ObjectID, ObjectNotAvailable, # noqa PlasmaBuffer, PlasmaClient, connect, @@ -84,7 +85,11 @@ def start_plasma_store(plasma_store_memory, plasma_directory=None, use_hugepages=False, external_store=None): """ - Start a plasma store process. + DEPRECATED: Start a plasma store process. + + .. deprecated:: 10.0.0 + Plasma is deprecated since Arrow 10.0.0. It will be removed + in 12.0.0 or so. Parameters ---------- @@ -109,6 +114,11 @@ def start_plasma_store(plasma_store_memory, A tuple of the name of the plasma store socket and the process ID of the plasma store process. """ + warnings.warn( + "Plasma is deprecated since Arrow 10.0.0. It will be removed in " + "12.0.0 or so.", + DeprecationWarning) + if use_valgrind and use_profiler: raise Exception("Cannot use valgrind and profiler at the same time.") diff --git a/python/pyarrow/tests/test_plasma.py b/python/pyarrow/tests/test_plasma.py index ed08a687258..cc5fd0357dd 100644 --- a/python/pyarrow/tests/test_plasma.py +++ b/python/pyarrow/tests/test_plasma.py @@ -30,6 +30,10 @@ import pyarrow as pa +# ignore all Plasma deprecation warnings in this file, we test that the +# warnings are actually raised in test_plasma_deprecated.py +pytestmark = pytest.mark.filterwarnings("ignore:Plasma:DeprecationWarning") + DEFAULT_PLASMA_STORE_MEMORY = 10 ** 8 USE_VALGRIND = os.getenv("PLASMA_VALGRIND") == "1" EXTERNAL_STORE = "hashtable://test" @@ -1071,3 +1075,27 @@ def test_store_capacity(): with plasma.start_plasma_store(plasma_store_memory=10000) as (name, p): plasma_client = plasma.connect(name) assert plasma_client.store_capacity() == 10000 + + +@pytest.mark.plasma +def test_plasma_deprecated(): + import pyarrow.plasma as plasma + + plasma_store_ctx = plasma.start_plasma_store( + plasma_store_memory=10 ** 8, + use_valgrind=os.getenv("PLASMA_VALGRIND") == "1") + + with pytest.warns(DeprecationWarning): + with plasma_store_ctx: + pass + + plasma_store_ctx = plasma.start_plasma_store( + plasma_store_memory=10 ** 8, + use_valgrind=os.getenv("PLASMA_VALGRIND") == "1") + + with plasma_store_ctx as (plasma_store_name, _): + with pytest.warns(DeprecationWarning): + plasma.connect(plasma_store_name) + + with pytest.warns(DeprecationWarning): + plasma.ObjectID(20 * b"a")