diff --git a/distributed/shuffle/_arrow.py b/distributed/shuffle/_arrow.py index a1068fce1f2..d2ac9ed96a2 100644 --- a/distributed/shuffle/_arrow.py +++ b/distributed/shuffle/_arrow.py @@ -56,7 +56,6 @@ def convert_partition(data: bytes, meta: pd.DataFrame) -> pd.DataFrame: sr = pa.RecordBatchStreamReader(file) shards.append(sr.read_all()) table = pa.concat_tables(shards, promote=True) - df = table.to_pandas(self_destruct=True) def default_types_mapper(pyarrow_dtype: pa.DataType) -> object: # Avoid converting strings from `string[pyarrow]` to `string[python]`