From 7ce1a54e0a2c2ce2c17340c54211572a47ed6d56 Mon Sep 17 00:00:00 2001 From: Patrick Hoefler <61934744+phofl@users.noreply.github.com> Date: Wed, 20 Sep 2023 16:27:23 +0200 Subject: [PATCH] Remove predicate pushdown from read_parquet --- dask_expr/io/parquet.py | 11 ----------- dask_expr/io/tests/test_io.py | 2 ++ 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/dask_expr/io/parquet.py b/dask_expr/io/parquet.py index 104c4c4bf..4bdbd2c6c 100644 --- a/dask_expr/io/parquet.py +++ b/dask_expr/io/parquet.py @@ -37,7 +37,6 @@ And, Blockwise, Expr, - Filter, Index, Lengths, Literal, @@ -457,16 +456,6 @@ def _simplify_up(self, parent): if isinstance(parent, Projection): return super()._simplify_up(parent) - if isinstance(parent, Filter) and isinstance( - parent.predicate, (LE, GE, LT, GT, EQ, NE, And, Or) - ): - # Predicate pushdown - filters = _DNF.extract_pq_filters(self, parent.predicate) - if filters: - kwargs = dict(zip(self._parameters, self.operands)) - kwargs["filters"] = filters.combine(kwargs["filters"]).to_list_tuple() - return ReadParquet(**kwargs) - if isinstance(parent, Lengths): _lengths = self._get_lengths() if _lengths: diff --git a/dask_expr/io/tests/test_io.py b/dask_expr/io/tests/test_io.py index c28c4a53e..ca00d9060 100644 --- a/dask_expr/io/tests/test_io.py +++ b/dask_expr/io/tests/test_io.py @@ -85,6 +85,7 @@ def test_io_fusion(tmpdir, fmt): assert_eq(df2, df[["a", "b"]] + 1) +@pytest.mark.skip() def test_predicate_pushdown(tmpdir): original = lib.DataFrame( { @@ -112,6 +113,7 @@ def test_predicate_pushdown(tmpdir): assert (y_result == 4).all() +@pytest.mark.skip() def test_predicate_pushdown_compound(tmpdir): pdf = lib.DataFrame( {