From d7938651c833111167735662f5c7a48562a8c182 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Mon, 10 Jul 2023 17:50:04 +0200 Subject: [PATCH] Remove accidental duplicated conversion of pyarrow Table to pandas --- distributed/shuffle/_arrow.py | 1 - 1 file changed, 1 deletion(-) diff --git a/distributed/shuffle/_arrow.py b/distributed/shuffle/_arrow.py index a1068fce1f2..d2ac9ed96a2 100644 --- a/distributed/shuffle/_arrow.py +++ b/distributed/shuffle/_arrow.py @@ -56,7 +56,6 @@ def convert_partition(data: bytes, meta: pd.DataFrame) -> pd.DataFrame: sr = pa.RecordBatchStreamReader(file) shards.append(sr.read_all()) table = pa.concat_tables(shards, promote=True) - df = table.to_pandas(self_destruct=True) def default_types_mapper(pyarrow_dtype: pa.DataType) -> object: # Avoid converting strings from `string[pyarrow]` to `string[python]`