Skip to content

[Fix](Nereids) TopN should forbid two-phase topN if child is DistributionSpecGather #36877

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morrySnow merged 5 commits into from
Jul 3, 2024
Merged

[Fix](Nereids) TopN should forbid two-phase topN if child is DistributionSpecGather #36877

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
95d0927
If child is DistributionSpecGather, topN should forbid two-phase topN
xiedeyantu Jun 26, 2024
4d46832
fix
xiedeyantu Jun 26, 2024
6e76e7f
chang order_push_down and push_down_top_n_through_union test
xiedeyantu Jun 27, 2024
fc1e2eb
update shape
xiedeyantu Jul 3, 2024
439aaa4
revert query51.out
xiedeyantu Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions ...e-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.JoinUtils;
Expand Down Expand Up @@ -476,6 +477,19 @@ public Boolean visitAbstractPhysicalSort(AbstractPhysicalSort<? extends Plan> so
return true;
}

@Override
public Boolean visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, Void context) {
// process must shuffle
visit(topN, context);

// If child is DistributionSpecGather, topN should forbid two-phase topN
if (topN.getSortPhase() == SortPhase.LOCAL_SORT
&& childrenProperties.get(0).getDistributionSpec().equals(DistributionSpecGather.INSTANCE)) {
return false;
}
return true;
}

/**
* check both side real output hash key order are same or not.
*
Expand Down
138 changes: 63 additions & 75 deletions regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,12 @@ PhysicalResultSink

-- !cross_join_order --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------NestedLoopJoin[CROSS_JOIN]
--------PhysicalTopN[MERGE_SORT]
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--------PhysicalOlapScan[t2]
--PhysicalTopN[GATHER_SORT]
----NestedLoopJoin[CROSS_JOIN]
------PhysicalTopN[MERGE_SORT]
--------PhysicalTopN[LOCAL_SORT]
----------PhysicalOlapScan[t1]
------PhysicalOlapScan[t2]

-- !limit_offset_sort_join --
PhysicalResultSink
Expand Down Expand Up @@ -172,12 +171,11 @@ PhysicalResultSink

-- !limit_sort_window --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalWindow
--------PhysicalQuickSort[MERGE_SORT]
----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--PhysicalTopN[GATHER_SORT]
----PhysicalWindow
------PhysicalQuickSort[MERGE_SORT]
--------PhysicalQuickSort[LOCAL_SORT]
----------PhysicalOlapScan[t1]

-- !limit_offset_window --
PhysicalResultSink
Expand All @@ -191,12 +189,11 @@ PhysicalResultSink

-- !limit_offset_sort_window --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalWindow
--------PhysicalQuickSort[MERGE_SORT]
----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--PhysicalTopN[GATHER_SORT]
----PhysicalWindow
------PhysicalQuickSort[MERGE_SORT]
--------PhysicalQuickSort[LOCAL_SORT]
----------PhysicalOlapScan[t1]

-- !limit_sort_filter --
PhysicalResultSink
Expand Down Expand Up @@ -264,21 +261,19 @@ PhysicalResultSink

-- !limit_window --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalWindow
--------PhysicalQuickSort[MERGE_SORT]
----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--PhysicalTopN[GATHER_SORT]
----PhysicalWindow
------PhysicalQuickSort[MERGE_SORT]
--------PhysicalQuickSort[LOCAL_SORT]
----------PhysicalOlapScan[t1]

-- !limit_offset_window --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalWindow
--------PhysicalQuickSort[MERGE_SORT]
----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--PhysicalTopN[GATHER_SORT]
----PhysicalWindow
------PhysicalQuickSort[MERGE_SORT]
--------PhysicalQuickSort[LOCAL_SORT]
----------PhysicalOlapScan[t1]

-- !limit_filter --
PhysicalResultSink
Expand Down Expand Up @@ -334,12 +329,11 @@ PhysicalResultSink

-- !limit_subquery_window --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalWindow
--------PhysicalQuickSort[MERGE_SORT]
----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--PhysicalTopN[GATHER_SORT]
----PhysicalWindow
------PhysicalQuickSort[MERGE_SORT]
--------PhysicalQuickSort[LOCAL_SORT]
----------PhysicalOlapScan[t1]

-- !limit_nested_subquery --
PhysicalResultSink
Expand Down Expand Up @@ -384,18 +378,16 @@ PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalUnion
--------PhysicalTopN[MERGE_SORT]
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalWindow
--------------PhysicalQuickSort[MERGE_SORT]
----------------PhysicalQuickSort[LOCAL_SORT]
------------------PhysicalOlapScan[t1]
--------PhysicalTopN[MERGE_SORT]
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalWindow
--------------PhysicalQuickSort[MERGE_SORT]
----------------PhysicalQuickSort[LOCAL_SORT]
------------------PhysicalOlapScan[t2]
--------PhysicalTopN[GATHER_SORT]
----------PhysicalWindow
------------PhysicalQuickSort[MERGE_SORT]
--------------PhysicalQuickSort[LOCAL_SORT]
----------------PhysicalOlapScan[t1]
--------PhysicalTopN[GATHER_SORT]
----------PhysicalWindow
------------PhysicalQuickSort[MERGE_SORT]
--------------PhysicalQuickSort[LOCAL_SORT]
----------------PhysicalOlapScan[t2]

-- !limit_subquery_join_filter --
PhysicalResultSink
Expand Down Expand Up @@ -533,12 +525,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----filter((t1.id < 10))
------PhysicalOlapScan[t1]
--PhysicalResultSink
----PhysicalTopN[MERGE_SORT]
------PhysicalTopN[LOCAL_SORT]
--------PhysicalWindow
----------PhysicalQuickSort[MERGE_SORT]
------------PhysicalQuickSort[LOCAL_SORT]
--------------PhysicalCteConsumer ( cteId=CTEId#0 )
----PhysicalTopN[GATHER_SORT]
------PhysicalWindow
--------PhysicalQuickSort[MERGE_SORT]
----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalCteConsumer ( cteId=CTEId#0 )

-- !limit_project_filter --
PhysicalResultSink
Expand Down Expand Up @@ -567,12 +558,11 @@ PhysicalResultSink

-- !limit_subquery_window --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalWindow
--------PhysicalQuickSort[MERGE_SORT]
----------PhysicalQuickSort[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--PhysicalTopN[GATHER_SORT]
----PhysicalWindow
------PhysicalQuickSort[MERGE_SORT]
--------PhysicalQuickSort[LOCAL_SORT]
----------PhysicalOlapScan[t1]

-- !limit_nested_subquery --
PhysicalResultSink
Expand All @@ -589,13 +579,12 @@ PhysicalResultSink

-- !limit_cross_join --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------NestedLoopJoin[CROSS_JOIN]
--------PhysicalTopN[MERGE_SORT]
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--------PhysicalOlapScan[t2]
--PhysicalTopN[GATHER_SORT]
----NestedLoopJoin[CROSS_JOIN]
------PhysicalTopN[MERGE_SORT]
--------PhysicalTopN[LOCAL_SORT]
----------PhysicalOlapScan[t1]
------PhysicalOlapScan[t2]

-- !limit_multiple_left_outer_join --
PhysicalResultSink
Expand Down Expand Up @@ -633,13 +622,12 @@ PhysicalResultSink

-- !limit_subquery_cross_join --
PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------NestedLoopJoin[CROSS_JOIN]
--------PhysicalTopN[MERGE_SORT]
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalOlapScan[t1]
--------PhysicalOlapScan[t2]
--PhysicalTopN[GATHER_SORT]
----NestedLoopJoin[CROSS_JOIN]
------PhysicalTopN[MERGE_SORT]
--------PhysicalTopN[LOCAL_SORT]
----------PhysicalOlapScan[t1]
------PhysicalOlapScan[t2]

-- !limit_subquery_multiple_join --
PhysicalResultSink
Expand Down
18 changes: 8 additions & 10 deletions regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,16 +106,14 @@ PhysicalResultSink
--PhysicalTopN[MERGE_SORT]
----PhysicalTopN[LOCAL_SORT]
------PhysicalUnion
--------PhysicalTopN[MERGE_SORT]
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalLimit[GLOBAL]
--------------PhysicalLimit[LOCAL]
----------------PhysicalOlapScan[table1]
--------PhysicalTopN[MERGE_SORT]
----------PhysicalTopN[LOCAL_SORT]
------------PhysicalLimit[GLOBAL]
--------------PhysicalLimit[LOCAL]
----------------PhysicalOlapScan[table1]
--------PhysicalTopN[GATHER_SORT]
----------PhysicalLimit[GLOBAL]
------------PhysicalLimit[LOCAL]
--------------PhysicalOlapScan[table1]
--------PhysicalTopN[GATHER_SORT]
----------PhysicalLimit[GLOBAL]
------------PhysicalLimit[LOCAL]
--------------PhysicalOlapScan[table1]

-- !push_down_topn_union_complex_conditions --
PhysicalResultSink
Expand Down
Loading