From 9371d86cce8514c029ab266f8c709043b371cfa9 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Fri, 14 Feb 2025 10:48:00 +0800 Subject: [PATCH] branch-3.1: [opt](Nereids) support defer materialization with project pick from #47661 #47975 #48747 --- .../org/apache/doris/catalog/OlapTable.java | 7 +- .../translator/PhysicalPlanTranslator.java | 49 ++-- .../rewrite/DeferMaterializeTopNResult.java | 251 ++++++++++++++++-- .../limit_push_down/order_push_down.out | 32 +-- .../lazy_materialize_topn.groovy | 73 +++++ .../limit_push_down/order_push_down.groovy | 2 +- .../single_table_without_aggregate.groovy | 3 + .../suites/query_p0/sort/topn_2pr_rule.groovy | 2 +- 8 files changed, 352 insertions(+), 67 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 1248cbd55372fc..00889d6336c74a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -94,6 +94,7 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; import lombok.Setter; +import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -2900,11 +2901,13 @@ public AutoIncrementGenerator getAutoIncrementGenerator() { * @param selectedIndexId the index want to scan */ public TFetchOption generateTwoPhaseReadOption(long selectedIndexId) { + boolean useStoreRow = this.storeRowColumn() + && CollectionUtils.isEmpty(getTableProperty().getCopiedRowStoreColumns()); TFetchOption fetchOption = new TFetchOption(); - fetchOption.setFetchRowStore(this.storeRowColumn()); + fetchOption.setFetchRowStore(useStoreRow); fetchOption.setUseTwoPhaseFetch(true); fetchOption.setNodesInfo(SystemInfoService.createAliveNodesInfo()); - if (!this.storeRowColumn()) { + if (!useStoreRow) { List columnsDesc = Lists.newArrayList(); getColumnDesc(selectedIndexId, columnsDesc, null, null); fetchOption.setColumnDesc(columnsDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 8ea9de280d6f4d..ca93b52772ee71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1936,7 +1936,13 @@ public PlanFragment visitPhysicalProject(PhysicalProject project List allProjectionExprs = Lists.newArrayList(); List slots = null; // TODO FE/BE do not support multi-layer-project on MultiDataSink now. - if (project.hasMultiLayerProjection() && !(inputFragment instanceof MultiCastPlanFragment)) { + if (project.hasMultiLayerProjection() + && !(inputFragment instanceof MultiCastPlanFragment) + // TODO support for two phase read with project, remove it after refactor + && !(project.child() instanceof PhysicalDeferMaterializeTopN) + && !(project.child() instanceof PhysicalDeferMaterializeOlapScan + || (project.child() instanceof PhysicalFilter + && ((PhysicalFilter) project.child()).child() instanceof PhysicalDeferMaterializeOlapScan))) { int layerCount = project.getMultiLayerProjects().size(); for (int i = 0; i < layerCount; i++) { List layer = project.getMultiLayerProjects().get(i); @@ -2044,37 +2050,28 @@ public PlanFragment visitPhysicalProject(PhysicalProject project } if (inputPlanNode instanceof ScanNode) { - TupleDescriptor projectionTuple = null; - // slotIdsByOrder is used to ensure the ScanNode's output order is same with current Project - // if we change the output order in translate project, the upper node will receive wrong order - // tuple, since they get the order from project.getOutput() not scan.getOutput()./ - projectionTuple = generateTupleDesc(slots, - ((ScanNode) inputPlanNode).getTupleDesc().getTable(), context); - inputPlanNode.setProjectList(projectionExprs); - inputPlanNode.setOutputTupleDesc(projectionTuple); - - // TODO: this is a temporary scheme to support two phase read when has project. - // we need to refactor all topn opt into rbo stage. + // TODO support for two phase read with project, remove this if after refactor + if (!(project.child() instanceof PhysicalDeferMaterializeOlapScan + || (project.child() instanceof PhysicalFilter + && ((PhysicalFilter) project.child()).child() instanceof PhysicalDeferMaterializeOlapScan))) { + TupleDescriptor projectionTuple = generateTupleDesc(slots, + ((ScanNode) inputPlanNode).getTupleDesc().getTable(), context); + inputPlanNode.setProjectList(projectionExprs); + inputPlanNode.setOutputTupleDesc(projectionTuple); + } if (inputPlanNode instanceof OlapScanNode) { - ArrayList olapScanSlots = - context.getTupleDesc(inputPlanNode.getTupleIds().get(0)).getSlots(); - SlotDescriptor lastSlot = olapScanSlots.get(olapScanSlots.size() - 1); - if (lastSlot.getColumn() != null - && lastSlot.getColumn().getName().equals(Column.ROWID_COL)) { - injectRowIdColumnSlot(projectionTuple); - SlotRef slotRef = new SlotRef(lastSlot); - inputPlanNode.getProjectList().add(slotRef); - requiredByProjectSlotIdSet.add(lastSlot.getId()); - requiredSlotIdSet.add(lastSlot.getId()); - } ((OlapScanNode) inputPlanNode).updateRequiredSlots(context, requiredByProjectSlotIdSet); } updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet, requiredByProjectSlotIdSet, context); } else { - TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context); - inputPlanNode.setProjectList(projectionExprs); - inputPlanNode.setOutputTupleDesc(tupleDescriptor); + if (project.child() instanceof PhysicalDeferMaterializeTopN) { + inputFragment.setOutputExprs(allProjectionExprs); + } else { + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context); + inputPlanNode.setProjectList(projectionExprs); + inputPlanNode.setOutputTupleDesc(tupleDescriptor); + } } return inputFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java index 6d90d81349d08f..765081ae016afe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java @@ -33,17 +33,21 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * rewrite simple top n query to defer materialize slot not use for sort or predicate @@ -54,51 +58,256 @@ public class DeferMaterializeTopNResult implements RewriteRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink(logicalTopN(logicalOlapScan())) - .when(r -> r.child().getLimit() < getTopNOptLimitThreshold()) - .whenNot(r -> r.child().getOrderKeys().isEmpty()) - .when(r -> r.child().getOrderKeys().stream().map(OrderKey::getExpr) - .allMatch(Expression::isColumnFromTable)) - .when(r -> r.child().child().getTable().getEnableLightSchemaChange()) - .when(r -> r.child().child().getTable().isDupKeysOrMergeOnWrite()) - .then(r -> deferMaterialize(r, r.child(), Optional.empty(), r.child().child())) + logicalResultSink( + logicalTopN( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> t.getOrderKeys().stream() + .map(OrderKey::getExpr) + .allMatch(Expression::isColumnFromTable)) + ).then(r -> deferMaterialize(r, r.child(), + Optional.empty(), Optional.empty(), r.child().child())) ), RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink(logicalTopN(logicalFilter(logicalOlapScan()))) - .when(r -> r.child().getLimit() < getTopNOptLimitThreshold()) - .whenNot(r -> r.child().getOrderKeys().isEmpty()) - .when(r -> r.child().getOrderKeys().stream().map(OrderKey::getExpr) - .allMatch(Expression::isColumnFromTable)) - .when(r -> r.child().child().child().getTable().getEnableLightSchemaChange()) - .when(r -> r.child().child().child().getTable().isDupKeysOrMergeOnWrite()) - .then(r -> { - LogicalFilter filter = r.child().child(); - return deferMaterialize(r, r.child(), Optional.of(filter), filter.child()); - }) + logicalResultSink( + logicalTopN( + logicalProject( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).then(r -> { + LogicalProject project = r.child().child(); + return deferMaterialize(r, r.child(), Optional.of(project), + Optional.empty(), project.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink( + logicalTopN( + logicalFilter( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> t.getOrderKeys().stream() + .map(OrderKey::getExpr) + .allMatch(Expression::isColumnFromTable)) + ).then(r -> { + LogicalFilter filter = r.child().child(); + return deferMaterialize(r, r.child(), Optional.empty(), + Optional.of(filter), filter.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink( + logicalTopN( + logicalProject( + logicalFilter( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).then(r -> { + LogicalProject> project = r.child().child(); + LogicalFilter filter = project.child(); + return deferMaterialize(r, r.child(), Optional.of(project), + Optional.of(filter), filter.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink(logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> { + LogicalProject upperProject = r.child(); + LogicalProject bottomProject = r.child().child().child(); + List projections = upperProject.mergeProjections(bottomProject); + LogicalProject project = upperProject.withProjects(projections); + return deferMaterialize(r, r.child().child(), Optional.of(project), + Optional.empty(), bottomProject.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink(logicalProject( + logicalTopN( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + } + return true; + }) + )).then(r -> deferMaterialize(r, r.child().child(), Optional.of(r.child()), + Optional.empty(), r.child().child().child())) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink(logicalProject( + logicalTopN( + logicalProject(logicalFilter( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> { + LogicalProject upperProject = r.child(); + LogicalProject> bottomProject = r.child().child().child(); + List projections = upperProject.mergeProjections(bottomProject); + LogicalProject project = upperProject.withProjects(projections); + LogicalFilter filter = bottomProject.child(); + return deferMaterialize(r, r.child().child(), Optional.of(project), + Optional.of(filter), filter.child()); + }) ) ); } private Plan deferMaterialize(LogicalResultSink logicalResultSink, - LogicalTopN logicalTopN, Optional> logicalFilter, - LogicalOlapScan logicalOlapScan) { + LogicalTopN logicalTopN, Optional> logicalProject, + Optional> logicalFilter, LogicalOlapScan logicalOlapScan) { Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column"); SlotReference columnId = SlotReference.fromColumn( logicalOlapScan.getTable(), rowId, logicalOlapScan.getQualifier()); + Set orderKeys = Sets.newHashSet(); Set deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet()); logicalFilter.ifPresent(filter -> filter.getConjuncts() .forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds()))); logicalTopN.getOrderKeys().stream() .map(OrderKey::getExpr) .map(Slot.class::cast) + .peek(orderKeys::add) .map(NamedExpression::getExprId) .filter(Objects::nonNull) .forEach(deferredMaterializedExprIds::remove); + if (logicalProject.isPresent()) { + deferredMaterializedExprIds.retainAll(logicalProject.get().getInputSlots().stream() + .map(NamedExpression::getExprId).collect(Collectors.toSet())); + } + if (deferredMaterializedExprIds.isEmpty()) { + // nothing to deferred materialize + return null; + } LogicalDeferMaterializeOlapScan deferOlapScan = new LogicalDeferMaterializeOlapScan( logicalOlapScan, deferredMaterializedExprIds, columnId); Plan root = logicalFilter.map(f -> f.withChildren(deferOlapScan)).orElse(deferOlapScan); + Set inputSlots = Sets.newHashSet(); + logicalFilter.ifPresent(filter -> inputSlots.addAll(filter.getInputSlots())); + if (logicalProject.isPresent()) { + ImmutableList.Builder requiredSlots = ImmutableList.builder(); + inputSlots.addAll(logicalProject.get().getInputSlots()); + for (Slot output : root.getOutput()) { + if (inputSlots.contains(output) || orderKeys.contains(output)) { + requiredSlots.add(output); + } + } + requiredSlots.add(columnId); + root = new LogicalProject<>(requiredSlots.build(), root); + } root = new LogicalDeferMaterializeTopN<>((LogicalTopN) logicalTopN.withChildren(root), deferredMaterializedExprIds, columnId); + if (logicalProject.isPresent()) { + // generate projections with the order exactly same as result output's + Map projectsMap = Maps.newHashMap(); + logicalProject.get().getProjects().forEach(p -> projectsMap.put(p.toSlot(), p)); + List outputProjects = logicalResultSink.getOutput().stream() + .map(projectsMap::get) + .collect(ImmutableList.toImmutableList()); + root = logicalProject.get().withProjectsAndChild(outputProjects, root); + } root = logicalResultSink.withChildren(root); return new LogicalDeferMaterializeResultSink<>((LogicalResultSink) root, logicalOlapScan.getTable(), logicalOlapScan.getSelectedIndexId()); diff --git a/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out b/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out index 694e609eaeb2af..5f0e3fa80998db 100644 --- a/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out +++ b/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out @@ -195,18 +195,18 @@ PhysicalResultSink ----------PhysicalOlapScan[t1] -- !limit_sort_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_offset_sort_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_subquery_order_by_inside_limit_outside -- PhysicalResultSink @@ -275,18 +275,18 @@ PhysicalResultSink ----------PhysicalOlapScan[t1] -- !limit_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_offset_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_project_filter -- PhysicalResultSink diff --git a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy new file mode 100644 index 00000000000000..cd683b08f280f2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("lazy_materialize_topn") { + sql """ + set enable_two_phase_read_opt = true + """ + + sql """ + drop table if exists lazy_materialize_topn; + """ + + sql """ + CREATE TABLE `lazy_materialize_topn` ( + `c1` int NULL, + `c2` int NULL, + `c3` int NULL, + `c4` array NULL + ) + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "light_schema_change" = "true" + ); + """ + + sql """ + insert into lazy_materialize_topn values (1, 1, 1, [1]), (2, 2, 2, [2]), (3, 3, 3, [3]); + """ + + sql """ + sync + """ + + List sqls = [ + // TopN(Scan) + """select * from lazy_materialize_topn order by c1 limit 10""", + // TopN(Project(Scan)) + """select c1, c2 from lazy_materialize_topn order by c1 limit 10""", + // Project(TopN(Scan)) + """select c1, c2, c3, c4 from lazy_materialize_topn order by c1 limit 10""", + // Project(TopN(Project(Scan))) + """select c1 + 1, c2 + 1 from (select c1, c2 from lazy_materialize_topn order by c1 limit 10) t""", + // TopN(Filter(Scan)) + """select * from lazy_materialize_topn where c2 < 5 order by c1 limit 10;""", + // TopN(Project(Filter(Scan))) + """select c1, c2, c3 from lazy_materialize_topn where c2 < 5 order by c1 limit 10;""", + // Project(TopN(Project(Filter(Scan)))) + """select c1 + 1, c2 + 1, c3 + 1 from ( select c1, c2, c3 from lazy_materialize_topn where c2 < 5 order by c1 limit 10) t""", + // project set is diff with output list + """select c1, c1, c2 from (select c1, c2 from lazy_materialize_topn where c3 < 1 order by c2 limit 1)t;""" + ] + + for (sqlStr in sqls) { + explain { + sql """${sqlStr}""" + contains """OPT TWO PHASE""" + } + sql """${sqlStr}""" + } +} diff --git a/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy b/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy index 1ae008df588a82..bbf4a64f1180a7 100644 --- a/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy @@ -26,7 +26,7 @@ suite("order_push_down") { sql 'set be_number_for_test=3' sql "set disable_nereids_rules='push_down_top_n_distinct_through_union'" sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - + sql 'set enable_two_phase_read_opt = true' //`limit 1 offset 1 + sort, project`: qt_limit_offset_sort_project """ explain shape plan SELECT t1.id FROM t1 ORDER BY id LIMIT 1 OFFSET 1; """ diff --git a/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy b/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy index 450fb9c0ea3187..3a07a959511390 100644 --- a/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy @@ -21,6 +21,9 @@ suite("single_table_without_aggregate") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "set enable_materialized_view_rewrite=true" + // TODO remove this variable after mv rewrite support defer materialized nodes + sql 'set enable_two_phase_read_opt = false' + sql """ drop table if exists orders diff --git a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy b/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy index 4b74cdad8538e4..0df8ab260309c9 100644 --- a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy +++ b/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy @@ -52,7 +52,7 @@ suite("topn_2pr_rule") { } else if("${key_type}" == "UNIQUE") { explain { sql("select * from ${table_name} order by k limit 1;") - notContains "OPT TWO PHASE" + contains "OPT TWO PHASE" } } else if("${key_type}" == "AGGREGATE") { explain {