From baa6bc37a6ac18c4bc39af2b0346376367fdb85c Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Wed, 20 Dec 2023 20:43:25 +0800 Subject: [PATCH] support match logicalAggregate(logicalProject(logicalFilter(logicalOlapScan())) without agg --- .../mv/AbstractSelectMaterializedIndexRule.java | 14 +++++++------- .../mv/SelectMaterializedIndexWithAggregate.java | 15 +++++++++++++++ .../SelectMaterializedIndexWithoutAggregate.java | 6 +++--- regression-test/data/mv_p0/k1s2m3/k1s2m3.out | 3 +++ regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy | 7 +++++++ 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index c1550cb5bd5d5b..02c033a036206c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -108,7 +108,7 @@ protected boolean shouldSelectIndexWithoutAgg(LogicalOlapScan scan) { } } - protected boolean containAllRequiredColumns(MaterializedIndex index, LogicalOlapScan scan, + protected static boolean containAllRequiredColumns(MaterializedIndex index, LogicalOlapScan scan, Set requiredScanOutput, Set requiredExpr, Set predicateExpr) { OlapTable table = scan.getTable(); MaterializedIndexMeta meta = table.getIndexMetaByIndexId(index.getId()); @@ -177,7 +177,7 @@ protected static boolean containsAllColumn(Expression expression, Set mv * 1. find matching key prefix most. * 2. sort by row count, column count and index id. */ - protected long selectBestIndex( + protected static long selectBestIndex( List candidates, LogicalOlapScan scan, Set predicates) { @@ -212,7 +212,7 @@ protected long selectBestIndex( return CollectionUtils.isEmpty(sortedIndexIds) ? scan.getTable().getBaseIndexId() : sortedIndexIds.get(0); } - protected List matchPrefixMost( + protected static List matchPrefixMost( LogicalOlapScan scan, List candidate, Set predicates, @@ -238,7 +238,7 @@ protected List matchPrefixMost( * Filter the input conjuncts those can use prefix and split into 2 groups: is equal-to or non-equal-to predicate * when comparing the key column. */ - private Map> filterCanUsePrefixIndexAndSplitByEquality( + private static Map> filterCanUsePrefixIndexAndSplitByEquality( Set conjuncts, Map exprIdToColName) { return conjuncts.stream() .map(expr -> PredicateChecker.canUsePrefixIndex(expr, exprIdToColName)) @@ -332,7 +332,7 @@ private Optional check(Expression maybeSlot, Expression maybeConst) { /////////////////////////////////////////////////////////////////////////// // Matching key prefix /////////////////////////////////////////////////////////////////////////// - private List matchKeyPrefixMost( + private static List matchKeyPrefixMost( OlapTable table, List indexes, Set equalColumns, @@ -350,7 +350,7 @@ private List matchKeyPrefixMost( return collect.descendingMap().firstEntry().getValue(); } - private int indexKeyPrefixMatchCount( + private static int indexKeyPrefixMatchCount( OlapTable table, MaterializedIndex index, Set equalColNames, @@ -370,7 +370,7 @@ private int indexKeyPrefixMatchCount( return matchCount; } - protected boolean preAggEnabledByHint(LogicalOlapScan olapScan) { + protected static boolean preAggEnabledByHint(LogicalOlapScan olapScan) { return olapScan.getHints().stream().anyMatch("PREAGGOPEN"::equalsIgnoreCase); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 2cf0797a311ee7..b36afa35c41565 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; +import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule.SlotContext; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -266,6 +267,20 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); + if (result.indexId == scan.getTable().getBaseIndexId()) { + LogicalOlapScan mvPlanWithoutAgg = SelectMaterializedIndexWithoutAggregate.select(scan, + project::getInputSlots, filter::getConjuncts, + Stream.concat(filter.getExpressions().stream(), + project.getExpressions().stream()) + .collect(ImmutableSet.toImmutableSet())); + SlotContext slotContextWithoutAgg = generateBaseScanExprToMvExpr(mvPlanWithoutAgg); + + return agg.withChildren(new LogicalProject( + generateProjectsAlias(project.getOutput(), slotContextWithoutAgg), + new ReplaceExpressions(slotContextWithoutAgg).replace( + project.withChildren(filter.withChildren(mvPlanWithoutAgg)), + mvPlanWithoutAgg))); + } if (result.exprRewriteMap.isEmpty()) { return new LogicalProject<>( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java index 04ff23a360c4bb..7960dd73df9a8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java @@ -165,7 +165,7 @@ public List buildRules() { * @param predicatesSupplier Supplier to get pushdown predicates. * @return Result scan node. */ - private LogicalOlapScan select( + public static LogicalOlapScan select( LogicalOlapScan scan, Supplier> requiredScanOutputSupplier, Supplier> predicatesSupplier, @@ -237,7 +237,7 @@ private LogicalOlapScan select( } } - private boolean isSameDataType(LogicalOlapScan scan, long selectIndex, Set slots) { + private static boolean isSameDataType(LogicalOlapScan scan, long selectIndex, Set slots) { if (selectIndex != scan.getTable().getBaseIndexId()) { Map columnTypes = scan.getTable().getSchemaByIndexId(selectIndex).stream().collect(Collectors @@ -252,7 +252,7 @@ private boolean isSameDataType(LogicalOlapScan scan, long selectIndex, Set return true; } - private boolean indexHasAggregate(MaterializedIndex index, LogicalOlapScan scan) { + private static boolean indexHasAggregate(MaterializedIndex index, LogicalOlapScan scan) { return scan.getTable().getSchemaByIndexId(index.getId()) .stream() .anyMatch(Column::isAggregated); diff --git a/regression-test/data/mv_p0/k1s2m3/k1s2m3.out b/regression-test/data/mv_p0/k1s2m3/k1s2m3.out index b429cf77a32924..a0d14b46927ca2 100644 --- a/regression-test/data/mv_p0/k1s2m3/k1s2m3.out +++ b/regression-test/data/mv_p0/k1s2m3/k1s2m3.out @@ -68,6 +68,9 @@ 2 4 3 \N +-- !select_mv -- +0 + -- !select_star -- \N 4 \N d -4 -4 -4 d diff --git a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy index 70d63b905c3c99..8d33b107da2875 100644 --- a/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy +++ b/regression-test/suites/mv_p0/k1s2m3/k1s2m3.groovy @@ -107,5 +107,12 @@ suite ("k1s2m3") { } qt_select_mv "select k1,sum(k2*k3) from d_table group by k1 order by k1;" + createMV("create materialized view kdup321 as select k3,k2,k1 from d_table;") + explain { + sql("select count(k2) from d_table where k3 = 1;") + contains "(kdup321)" + } + qt_select_mv "select count(k2) from d_table where k3 = 1;" + qt_select_star "select * from d_table order by k1;" }