From 96f923e11cf5cb0a2091f6715e392ca5e05a1c1a Mon Sep 17 00:00:00 2001 From: Pxl Date: Tue, 23 Jan 2024 10:39:18 +0800 Subject: [PATCH 1/4] [Feature](materialized-view) support predicate apprear both on key and value mv column (#30215) support predicate apprear both on key and value mv column --- .../AbstractSelectMaterializedIndexRule.java | 3 +- .../SelectMaterializedIndexWithAggregate.java | 94 +++++-------------- .../testAggQueryOnAggMV1.out | 16 ++++ .../testAggQueryOnAggMV1.groovy | 28 +++++- 4 files changed, 70 insertions(+), 71 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 17eadbd319b958..26a12bb657e7b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -36,6 +36,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; @@ -160,7 +161,7 @@ protected static boolean containsAllColumn(Expression expression, Set mv return true; } if (expression.children().isEmpty()) { - return false; + return expression instanceof VirtualSlotReference; } for (Expression child : expression.children()) { if (child instanceof Literal) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 370d445049eaa5..bac8a17788c975 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -204,7 +204,7 @@ public List buildRules() { Optional.of(project)), ExpressionUtils.replace(agg.getGroupByExpressions(), project.getAliasToProducer()), - collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) + collectRequireExprWithAggAndProject(agg.getExpressions(), Optional.of(project)) ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); @@ -251,7 +251,7 @@ public List buildRules() { .collect(Collectors.toSet()); ImmutableSet requiredExpr = ImmutableSet.builder() .addAll(collectRequireExprWithAggAndProject( - agg.getExpressions(), project.getProjects())) + agg.getExpressions(), Optional.of(project))) .addAll(filter.getExpressions()) .build(); SelectResult result = select( @@ -306,9 +306,9 @@ public List buildRules() { LogicalOlapScan scan = project.child(); ImmutableSet requiredExpr = ImmutableSet.builder() .addAll(collectRequireExprWithAggAndProject( - agg.getExpressions(), project.getProjects())) + agg.getExpressions(), Optional.of(project))) .addAll(collectRequireExprWithAggAndProject( - filter.getExpressions(), project.getProjects())) + filter.getExpressions(), Optional.of(project))) .build(); SelectResult result = select( scan, @@ -468,7 +468,7 @@ public List buildRules() { Optional.of(project)), ExpressionUtils.replace(nonVirtualGroupByExprs(agg), project.getAliasToProducer()), - collectRequireExprWithAggAndProject(agg.getExpressions(), project.getProjects()) + collectRequireExprWithAggAndProject(agg.getExpressions(), Optional.of(project)) ); LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); @@ -522,7 +522,7 @@ public List buildRules() { .collect(Collectors.toSet()); ImmutableSet requiredExpr = ImmutableSet.builder() .addAll(collectRequireExprWithAggAndProject( - agg.getExpressions(), project.getProjects())) + agg.getExpressions(), Optional.of(project))) .addAll(filter.getExpressions()) .build(); SelectResult result = select( @@ -586,9 +586,9 @@ public List buildRules() { LogicalOlapScan scan = project.child(); ImmutableSet requiredExpr = ImmutableSet.builder() .addAll(collectRequireExprWithAggAndProject( - agg.getExpressions(), project.getProjects())) + agg.getExpressions(), Optional.of(project))) .addAll(collectRequireExprWithAggAndProject( - filter.getExpressions(), project.getProjects())) + filter.getExpressions(), Optional.of(project))) .build(); SelectResult result = select( scan, @@ -1070,41 +1070,10 @@ private AggRewriteResult rewriteAgg(MaterializedIndex index, ExprRewriteMap exprRewriteMap = new ExprRewriteMap(); RewriteContext context = new RewriteContext(new CheckContext(scan, index.getId()), exprRewriteMap); aggregateFunctions.forEach(aggFun -> AggFuncRewriter.rewrite(aggFun, context)); - - // has rewritten agg functions - Map slotMap = exprRewriteMap.slotMap; - // Note that the slots in the rewritten agg functions shouldn't appear in filters or grouping expressions. - // For example: we have a duplicated-type table t(c1, c2) and a materialized index that has - // a bitmap_union column `mv_bitmap_union_c2` for the column c2. - // The query `select c1, count(distinct c2) from t where c2 > 0 group by c1` can't use the materialized - // index because we have a filter `c2 > 0` for the aggregated column c2. - Set slotsToReplace = slotMap.keySet(); - Set indexConjuncts; - try { - indexConjuncts = PlanNode - .splitAndCompoundPredicateToConjuncts(context.checkContext.getMeta().getWhereClause()).stream() - .map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet()); - } catch (Exception e) { - return new AggRewriteResult(index, false, null, null); - } - if (isInputSlotsContainsNone( - predicates.stream().filter(e -> !indexConjuncts.contains(e.toSql())).collect(Collectors.toList()), - slotsToReplace) && isInputSlotsContainsNone(groupingExprs, slotsToReplace)) { - ImmutableSet newRequiredSlots = requiredScanOutput.stream() - .map(slot -> (Slot) ExpressionUtils.replace(slot, slotMap)).collect(ImmutableSet.toImmutableSet()); - return new AggRewriteResult(index, true, newRequiredSlots, exprRewriteMap); - } - - return new AggRewriteResult(index, false, null, null); + return new AggRewriteResult(index, true, requiredScanOutput, exprRewriteMap); } private static class ExprRewriteMap { - - /** - * Replace map for scan output slot. - */ - public final Map slotMap; - /** * Replace map for expressions in project. */ @@ -1117,13 +1086,12 @@ private static class ExprRewriteMap { private Map aggFuncStrMap; public ExprRewriteMap() { - this.slotMap = Maps.newHashMap(); this.projectExprMap = Maps.newHashMap(); this.aggFuncMap = Maps.newHashMap(); } public boolean isEmpty() { - return slotMap.isEmpty(); + return aggFuncMap.isEmpty(); } private void buildStrMap() { @@ -1224,7 +1192,6 @@ public Expression visitCount(Count count, RewriteContext context) { .orElseThrow(() -> new AnalysisException( "cannot find bitmap union slot when select mv")); - context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionSlot); context.exprRewriteMap.projectExprMap.put(slotOpt.get(), bitmapUnionSlot); BitmapUnionCount bitmapUnionCount = new BitmapUnionCount(bitmapUnionSlot); context.exprRewriteMap.aggFuncMap.put(count, bitmapUnionCount); @@ -1255,9 +1222,6 @@ public Expression visitCount(Count count, RewriteContext context) { .filter(s -> countColumn.equalsIgnoreCase(normalizeName(s.getName()))).findFirst() .orElseThrow(() -> new AnalysisException("cannot find count slot when select mv")); - if (child instanceof Slot) { - context.exprRewriteMap.slotMap.put((Slot) child, countSlot); - } context.exprRewriteMap.projectExprMap.put(child, countSlot); Sum sum = new Sum(countSlot); context.exprRewriteMap.aggFuncMap.put(count, sum); @@ -1294,7 +1258,6 @@ public Expression visitBitmapUnion(BitmapUnion bitmapUnion, RewriteContext conte .findFirst().orElseThrow( () -> new AnalysisException("cannot find bitmap union slot when select mv")); - context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionSlot); context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionSlot); BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot); context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion); @@ -1314,9 +1277,6 @@ public Expression visitBitmapUnion(BitmapUnion bitmapUnion, RewriteContext conte .stream().filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst() .orElseThrow(() -> new AnalysisException("cannot find bitmap union slot when select mv")); - if (child instanceof Slot) { - context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionSlot); - } context.exprRewriteMap.projectExprMap.put(child, bitmapUnionSlot); BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot); context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion); @@ -1356,7 +1316,6 @@ public Expression visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, Rewri .orElseThrow(() -> new AnalysisException( "cannot find bitmap union count slot when select mv")); - context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot); context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionCountSlot); BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot); context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount); @@ -1376,9 +1335,6 @@ public Expression visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, Rewri .stream().filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName()))) .findFirst().orElseThrow( () -> new AnalysisException("cannot find bitmap union count slot when select mv")); - if (child instanceof Slot) { - context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionCountSlot); - } context.exprRewriteMap.projectExprMap.put(child, bitmapUnionCountSlot); BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot); context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount); @@ -1415,7 +1371,6 @@ public Expression visitHllUnion(HllUnion hllUnion, RewriteContext context) { .orElseThrow(() -> new AnalysisException( "cannot find hll union slot when select mv")); - context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot); context.exprRewriteMap.projectExprMap.put(hllHash, hllUnionSlot); HllUnion newHllUnion = new HllUnion(hllUnionSlot); context.exprRewriteMap.aggFuncMap.put(hllUnion, newHllUnion); @@ -1453,7 +1408,6 @@ public Expression visitHllUnionAgg(HllUnionAgg hllUnionAgg, RewriteContext conte .orElseThrow(() -> new AnalysisException( "cannot find hll union slot when select mv")); - context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot); context.exprRewriteMap.projectExprMap.put(hllHash, hllUnionSlot); HllUnionAgg newHllUnionAgg = new HllUnionAgg(hllUnionSlot); context.exprRewriteMap.aggFuncMap.put(hllUnionAgg, newHllUnionAgg); @@ -1492,7 +1446,6 @@ public Expression visitNdv(Ndv ndv, RewriteContext context) { .orElseThrow(() -> new AnalysisException( "cannot find hll union slot when select mv")); - context.exprRewriteMap.slotMap.put(slotOpt.get(), hllUnionSlot); context.exprRewriteMap.projectExprMap.put(slotOpt.get(), hllUnionSlot); HllUnionAgg hllUnionAgg = new HllUnionAgg(hllUnionSlot); context.exprRewriteMap.aggFuncMap.put(ndv, hllUnionAgg); @@ -1518,7 +1471,6 @@ public Expression visitSum(Sum sum, RewriteContext context) { Slot sumSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index).stream() .filter(s -> sumColumn.equalsIgnoreCase(normalizeName(s.getName()))).findFirst() .orElseThrow(() -> new AnalysisException("cannot find sum slot when select mv")); - context.exprRewriteMap.slotMap.put(slotOpt.get(), sumSlot); context.exprRewriteMap.projectExprMap.put(sum.child(), sumSlot); Sum newSum = new Sum(sumSlot); context.exprRewriteMap.aggFuncMap.put(sum, newSum); @@ -1545,7 +1497,6 @@ public Expression visitAggregateFunction(AggregateFunction aggregateFunction, Re Set slots = aggregateFunction.collect(SlotReference.class::isInstance); for (Slot slot : slots) { - context.exprRewriteMap.slotMap.put(slot, aggStateSlot); context.exprRewriteMap.projectExprMap.put(slot, aggStateSlot); } @@ -1660,8 +1611,13 @@ private List generateNewOutputsWithMvOutputs( * +--LogicalOlapScan() * t -> abs(k1#0) + 1 */ - private Set collectRequireExprWithAggAndProject( - List aggExpressions, List projectExpressions) { + private Set collectRequireExprWithAggAndProject(List aggExpressions, + Optional> project) { + List projectExpressions = project.isPresent() ? project.get().getProjects() : null; + if (projectExpressions == null) { + return aggExpressions.stream().collect(ImmutableSet.toImmutableSet()); + } + Optional> slotToProducerOpt = project.map(Project::getAliasToProducer); Map exprIdToExpression = projectExpressions.stream() .collect(Collectors.toMap(NamedExpression::getExprId, e -> { if (e instanceof Alias) { @@ -1669,13 +1625,13 @@ private Set collectRequireExprWithAggAndProject( } return e; })); - return aggExpressions.stream() - .map(e -> { - if ((e instanceof NamedExpression) - && exprIdToExpression.containsKey(((NamedExpression) e).getExprId())) { - return exprIdToExpression.get(((NamedExpression) e).getExprId()); - } - return e; - }).collect(ImmutableSet.toImmutableSet()); + return aggExpressions.stream().map(e -> { + if ((e instanceof NamedExpression) && exprIdToExpression.containsKey(((NamedExpression) e).getExprId())) { + return exprIdToExpression.get(((NamedExpression) e).getExprId()); + } + return e; + }).map(e -> { + return slotToProducerOpt.map(slotToExpressions -> ExpressionUtils.replace(e, slotToExpressions)).orElse(e); + }).collect(ImmutableSet.toImmutableSet()); } } diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out index e64409afb2137d..f37d3ef54f31f6 100644 --- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out +++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out @@ -13,3 +13,19 @@ -- !select_mv -- 7 +-- !select_mv -- +1 2 +2 1 +3 1 + +-- !select_mv -- +1 2 +2 2 +3 3 + +-- !select_mv -- +1 2 + +-- !select_mv -- +1 2 1 + diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy index 044b92425ec0de..9b3e8ba371e328 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy @@ -37,7 +37,9 @@ suite ("testAggQueryOnAggMV1") { sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" - createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno ;") + createMV("create materialized view emps_mv as select deptno, sum(salary), max(commission) from emps group by deptno;") + createMV("create materialized view emps_mv_count_key as select deptno, count(deptno) from emps group by deptno;") + createMV("create materialized view emps_mv_if as select deptno, sum(if(empid = 1, empid, salary)) from emps group by deptno;") sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" @@ -59,4 +61,28 @@ suite ("testAggQueryOnAggMV1") { contains "(emps_mv)" } qt_select_mv "select sum(salary) as salary from emps;" + + explain { + sql("select deptno, count(deptno) from emps group by deptno order by deptno;") + contains "(emps_mv_count_key)" + } + qt_select_mv "select deptno, count(deptno) from emps group by deptno order by deptno;" + + explain { + sql("select deptno, sum(if(empid = 1, empid, salary)) from emps group by deptno;") + contains "(emps_mv_if)" + } + qt_select_mv "select deptno, sum(if(empid = 1, empid, salary)) from emps group by deptno order by deptno;" + + explain { + sql("select deptno, count(deptno) from emps where deptno=1 group by deptno order by deptno;") + contains "(emps_mv_count_key)" + } + qt_select_mv "select deptno, count(deptno) from emps where deptno=1 group by deptno order by deptno;" + + explain { + sql("select deptno, sum(salary), max(commission) from emps where salary=1 group by deptno order by deptno;") + contains "(emps)" + } + qt_select_mv "select deptno, sum(salary), max(commission) from emps where salary=1 group by deptno order by deptno;" } \ No newline at end of file From afda2cf5d5c79d811a767206071fc04eaa8bb207 Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 8 Mar 2024 11:06:37 +0800 Subject: [PATCH 2/4] [Feature](materialized-view) support mv with bitmap_union(bitmap_from_array()) case (#31962) support mv with bitmap_union(bitmap_from_array()) case --- .../apache/doris/analysis/ArrayLiteral.java | 4 +- .../SelectMaterializedIndexWithAggregate.java | 423 +++++++----------- .../data/mv_p0/test_mv_dp/test_mv_dp.out | 5 + .../suites/mv_p0/test_mv_dp/test_mv_dp.groovy | 71 +++ 4 files changed, 232 insertions(+), 271 deletions(-) create mode 100644 regression-test/data/mv_p0/test_mv_dp/test_mv_dp.out create mode 100644 regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index 53142aea7d6c34..dbbe0ce1ad120d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -100,7 +100,7 @@ protected String toSqlImpl() { List list = new ArrayList<>(children.size()); children.forEach(v -> list.add(v.toSqlImpl())); - return "ARRAY(" + StringUtils.join(list, ", ") + ")"; + return "[" + StringUtils.join(list, ", ") + "]"; } @Override @@ -108,7 +108,7 @@ public String toDigestImpl() { List list = new ArrayList<>(children.size()); children.forEach(v -> list.add(v.toDigestImpl())); - return "ARRAY(" + StringUtils.join(list, ", ") + ")"; + return "[" + StringUtils.join(list, ", ") + "]"; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index bac8a17788c975..29804285eb851a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -120,22 +120,17 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new ReplaceExpressions(slotContext) - .replace(agg.withChildren(mvPlan), mvPlan); - } else { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput( - agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - mvPlan - ), mvPlan)); - } + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput( + agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + mvPlan + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_SCAN), // filter could push down scan. @@ -166,27 +161,20 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren(filter.withChildren(mvPlan)), mvPlan)); - } else { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.empty(), Optional.empty(), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - // Note that no need to replace slots in the filter, - // because the slots to - // replace are value columns, which shouldn't appear in filters. - filter.withChildren(mvPlan) - ), mvPlan)); - } + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.empty(), Optional.empty(), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + // Note that no need to replace slots in the filter, + // because the slots to + // replace are value columns, which shouldn't appear in filters. + filter.withChildren(mvPlan) + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_SCAN), // column pruning or other projections such as alias, etc. @@ -210,32 +198,22 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( - project.withProjectsAndChild( - generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), - mvPlan)), mvPlan)); - } else { - List newProjectList = replaceProjectList(project, - result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>( - generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - newProject - ), mvPlan)); - } + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + newProject + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_SCAN), // filter could push down and project. @@ -267,33 +245,23 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( - project.withProjectsAndChild( - generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), - filter.withChildren(mvPlan))), mvPlan)); - } else { - List newProjectList = replaceProjectList(project, - result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>( - generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - filter.withChildren(mvPlan)); - - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - newProject - ), mvPlan)); - } + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + filter.withChildren(mvPlan)); + + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + newProject + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_FILTER_SCAN), // filter can't push down @@ -323,33 +291,22 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( - filter.withChildren( - project.withProjectsAndChild( - generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), - mvPlan))), mvPlan)); - } else { - List newProjectList = replaceProjectList(project, - result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>( - generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan); - - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - filter.withChildren(newProject) - ), mvPlan)); - } + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan); + + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + filter.withChildren(newProject) + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_PROJECT_SCAN), // only agg above scan @@ -370,28 +327,18 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput( + agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), repeat.withAggOutputAndChild( generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) ), mvPlan)); - } else { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput( - agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) - ), mvPlan)); - } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN), // filter could push down scan. @@ -423,33 +370,22 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.empty(), Optional.empty(), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), + // Not that no need to replace slots in the filter, + // because the slots to replace + // are value columns, which shouldn't appear in filters. repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - filter.withChildren(mvPlan) - )), mvPlan)); - } else { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.empty(), Optional.empty(), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - // Not that no need to replace slots in the filter, - // because the slots to replace - // are value columns, which shouldn't appear in filters. - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - filter.withChildren(mvPlan)) - ), mvPlan)); - } + generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), + filter.withChildren(mvPlan)) + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_SCAN), // column pruning or other projections such as alias, etc. @@ -474,37 +410,24 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + mvPlan); + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - project.withProjectsAndChild( - generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), - mvPlan)) + generateNewOutputsWithMvOutputs( + mvPlan, repeat.getOutputs()), newProject) ), mvPlan)); - } else { - List newProjectList = replaceProjectList(project, - result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>( - generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - mvPlan); - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs( - mvPlan, repeat.getOutputs()), newProject) - ), mvPlan)); - } }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_SCAN), // filter could push down and project. @@ -538,40 +461,25 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + filter.withChildren(mvPlan)); + + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - project.withProjectsAndChild( - generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), - filter.withChildren( - mvPlan - )))), - mvPlan)); - } else { - List newProjectList = replaceProjectList(project, - result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>( - generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - filter.withChildren(mvPlan)); - - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs( - mvPlan, repeat.getOutputs()), newProject) - ), mvPlan)); - } + generateNewOutputsWithMvOutputs( + mvPlan, repeat.getOutputs()), newProject) + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_FILTER_SCAN), // filter can't push down @@ -603,40 +511,25 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - if (result.exprRewriteMap.isEmpty()) { - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - agg.withChildren( + List newProjectList = replaceProjectList(project, + result.exprRewriteMap.projectExprMap); + LogicalProject newProject = new LogicalProject<>( + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), + scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); + + return new LogicalProject<>( + generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace( + new LogicalAggregate<>( + agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), + agg.getSourceRepeat(), repeat.withAggOutputAndChild( generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - filter.withChildren( - project.withProjectsAndChild( - generateNewOutputsWithMvOutputs(mvPlan, project.getProjects()), - mvPlan - )))), - mvPlan)); - } else { - List newProjectList = replaceProjectList(project, - result.exprRewriteMap.projectExprMap); - LogicalProject newProject = new LogicalProject<>( - generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); - - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - filter.withChildren(newProject)) - ), mvPlan)); - } + filter.withChildren(newProject)) + ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_PROJECT_SCAN) ); } @@ -702,7 +595,7 @@ private SelectResult select(LogicalOlapScan scan, Set requiredScanOutput, .filter(aggRewriteResult -> checkPreAggStatus(scan, aggRewriteResult.index.getId(), predicates, // check pre-agg status of aggregate function that couldn't rewrite. aggFuncsDiff(aggregateFunctions, aggRewriteResult), groupingExprs).isOn()) - .filter(result -> result.success).collect(Collectors.toList()); + .collect(Collectors.toList()); List haveAllRequiredColumns = Streams.concat( candidatesWithoutRewriting.stream() @@ -737,12 +630,8 @@ private SelectResult select(LogicalOlapScan scan, Set requiredScanOutput, private List aggFuncsDiff(List aggregateFunctions, AggRewriteResult aggRewriteResult) { - if (aggRewriteResult.success) { - return ImmutableList.copyOf(Sets.difference(ImmutableSet.copyOf(aggregateFunctions), - aggRewriteResult.exprRewriteMap.aggFuncMap.keySet())); - } else { - return aggregateFunctions; - } + return ImmutableList.copyOf(Sets.difference(ImmutableSet.copyOf(aggregateFunctions), + aggRewriteResult.exprRewriteMap.aggFuncMap.keySet())); } private static class SelectResult { @@ -876,8 +765,7 @@ public PreAggStatus visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, Che if (expr instanceof ToBitmap) { expr = expr.child(0); } - Optional slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(expr); - if (slotOpt.isPresent() && context.valueNameToColumn.containsKey(normalizeName(slotOpt.get().toSql()))) { + if (context.valueNameToColumn.containsKey(normalizeName(expr.toSql()))) { return PreAggStatus.on(); } else { return PreAggStatus.off("invalid bitmap_union_count: " + bitmapUnionCount.toSql()); @@ -1070,7 +958,7 @@ private AggRewriteResult rewriteAgg(MaterializedIndex index, ExprRewriteMap exprRewriteMap = new ExprRewriteMap(); RewriteContext context = new RewriteContext(new CheckContext(scan, index.getId()), exprRewriteMap); aggregateFunctions.forEach(aggFun -> AggFuncRewriter.rewrite(aggFun, context)); - return new AggRewriteResult(index, true, requiredScanOutput, exprRewriteMap); + return new AggRewriteResult(index, requiredScanOutput, exprRewriteMap); } private static class ExprRewriteMap { @@ -1118,16 +1006,13 @@ public Expression replaceAgg(Expression e) { private static class AggRewriteResult { public final MaterializedIndex index; - public final boolean success; public final Set requiredScanOutput; public ExprRewriteMap exprRewriteMap; public AggRewriteResult(MaterializedIndex index, - boolean success, Set requiredScanOutput, ExprRewriteMap exprRewriteMap) { this.index = index; - this.success = success; this.requiredScanOutput = requiredScanOutput; this.exprRewriteMap = exprRewriteMap; } diff --git a/regression-test/data/mv_p0/test_mv_dp/test_mv_dp.out b/regression-test/data/mv_p0/test_mv_dp/test_mv_dp.out new file mode 100644 index 00000000000000..27c89880a79797 --- /dev/null +++ b/regression-test/data/mv_p0/test_mv_dp/test_mv_dp.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv -- +1 4 4 +2 2 1 + diff --git a/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy new file mode 100644 index 00000000000000..dffb0825c53db4 --- /dev/null +++ b/regression-test/suites/mv_p0/test_mv_dp/test_mv_dp.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_mv_dp") { + + sql """ DROP TABLE IF EXISTS dp; """ + + sql """ + CREATE TABLE dp ( + `d` int NULL , + `status` text NULL , + `uid_list` array NULL + ) ENGINE=OLAP + DUPLICATE KEY(`d`) + DISTRIBUTED BY HASH(`d`) BUCKETS AUTO + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """INSERT INTO `dp` VALUES (1,'success',["1","2"]),(2,'fail',["1"]);""" + + createMV("""CREATE MATERIALIZED VIEW view_2 as + select d, + bitmap_union(bitmap_from_array(cast(uid_list as array))), + bitmap_union(bitmap_from_array(if(status='success', cast(uid_list as array), array()))) + from dp + group by d;""") + + sql """INSERT INTO `dp` VALUES (1,'success',["3","4"]),(2,'success',["5"]);""" +/* + streamLoad { + table "test" + + set 'columns', 'date' + + file './test' + time 10000 // limit inflight 10s + } +*/ + explain { + sql("""select d, + bitmap_union_count(bitmap_from_array(cast(uid_list as array))), + bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array), array()))) + from dp + group by d;""") + contains "(view_2)" + } + + qt_select_mv """select d, + bitmap_union_count(bitmap_from_array(cast(uid_list as array))), + bitmap_union_count(bitmap_from_array(if(status='success', cast(uid_list as array), array()))) + from dp + group by d order by 1;""" +} From 379637d38ce3adb500a1494a70076a24bfa8aef5 Mon Sep 17 00:00:00 2001 From: Pxl Date: Sun, 7 Apr 2024 10:53:29 +0800 Subject: [PATCH 3/4] [Bug](materialized-view) fix wrong result when salias name same with base slot on mv (#33198) fix wrong result when salias name same with base slot on mv --- .../AbstractSelectMaterializedIndexRule.java | 8 -- .../SelectMaterializedIndexWithAggregate.java | 118 ++++++++---------- .../test_upper_alias/test_upper_alias.out | 13 ++ .../test_dup_mv_year/test_dup_mv_year.groovy | 1 - .../test_upper_alias/test_upper_alias.groovy | 69 ++++++++++ 5 files changed, 136 insertions(+), 73 deletions(-) create mode 100644 regression-test/data/mv_p0/test_upper_alias/test_upper_alias.out create mode 100644 regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 26a12bb657e7b5..5ecb7171724a62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -416,9 +416,6 @@ protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan) { for (Slot mvSlot : mvPlan.getOutputByIndex(mvPlan.getSelectedIndexId())) { boolean isPushed = false; for (Slot baseSlot : mvPlan.getOutput()) { - if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) { - continue; - } if (baseSlot.toSql().equalsIgnoreCase( org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker( normalizeName(mvSlot.getName())))) { @@ -428,11 +425,6 @@ protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan) { } } if (!isPushed) { - if (org.apache.doris.analysis.CreateMaterializedViewStmt.isMVColumn(mvSlot.getName())) { - mvNameToMvSlot.put(normalizeName( - org.apache.doris.analysis.CreateMaterializedViewStmt.mvColumnBreaker(mvSlot.getName())), - mvSlot); - } mvNameToMvSlot.put(normalizeName(mvSlot.getName()), mvSlot); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 29804285eb851a..33709ca0175a25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -327,18 +327,16 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( + return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>( agg.getGroupByExpressions(), replaceAggOutput( - agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), + agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), + agg.isNormalized(), agg.getSourceRepeat(), repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), mvPlan) - ), mvPlan)); + replaceRepeatOutput(repeat, result.exprRewriteMap.projectExprMap), + mvPlan)), + mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN), // filter could push down scan. @@ -370,21 +368,18 @@ public List buildRules() { LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result); SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan); - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( + return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>( agg.getGroupByExpressions(), replaceAggOutput(agg, Optional.empty(), Optional.empty(), result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), + agg.isNormalized(), agg.getSourceRepeat(), // Not that no need to replace slots in the filter, // because the slots to replace // are value columns, which shouldn't appear in filters. repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - filter.withChildren(mvPlan)) + replaceRepeatOutput(repeat, result.exprRewriteMap.projectExprMap), + filter.withChildren(mvPlan)) ), mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_SCAN), @@ -413,21 +408,17 @@ public List buildRules() { List newProjectList = replaceProjectList(project, result.exprRewriteMap.projectExprMap); LogicalProject newProject = new LogicalProject<>( - generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - mvPlan); - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), + generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan); + + return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext), new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs( - mvPlan, repeat.getOutputs()), newProject) - ), mvPlan)); + new LogicalAggregate<>(agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), agg.getSourceRepeat(), + repeat.withAggOutputAndChild(replaceRepeatOutput(repeat, + result.exprRewriteMap.projectExprMap), newProject)), + mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_SCAN), // filter could push down and project. @@ -467,19 +458,15 @@ public List buildRules() { generateNewOutputsWithMvOutputs(mvPlan, newProjectList), filter.withChildren(mvPlan)); - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), + return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext), new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), - repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs( - mvPlan, repeat.getOutputs()), newProject) - ), mvPlan)); + new LogicalAggregate<>(agg.getGroupByExpressions(), + replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), + result.exprRewriteMap), + agg.isNormalized(), agg.getSourceRepeat(), + repeat.withAggOutputAndChild(replaceRepeatOutput(repeat, + result.exprRewriteMap.projectExprMap), newProject)), + mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_FILTER_SCAN), // filter can't push down @@ -517,19 +504,15 @@ public List buildRules() { generateNewOutputsWithMvOutputs(mvPlan, newProjectList), scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); - return new LogicalProject<>( - generateProjectsAlias(agg.getOutputs(), slotContext), - new ReplaceExpressions(slotContext).replace( - new LogicalAggregate<>( - agg.getGroupByExpressions(), - replaceAggOutput(agg, Optional.of(project), Optional.of(newProject), - result.exprRewriteMap), - agg.isNormalized(), - agg.getSourceRepeat(), + return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext), + new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>( + agg.getGroupByExpressions(), replaceAggOutput(agg, Optional.of(project), + Optional.of(newProject), result.exprRewriteMap), + agg.isNormalized(), agg.getSourceRepeat(), repeat.withAggOutputAndChild( - generateNewOutputsWithMvOutputs(mvPlan, repeat.getOutputs()), - filter.withChildren(newProject)) - ), mvPlan)); + replaceRepeatOutput(repeat, result.exprRewriteMap.projectExprMap), + filter.withChildren(newProject))), + mvPlan)); }).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_PROJECT_SCAN) ); } @@ -580,22 +563,22 @@ private SelectResult select(LogicalOlapScan scan, Set requiredScanOutput, .stream() .collect(Collectors.groupingBy(index -> index.getId() == table.getBaseIndexId())); - Set candidatesWithoutRewriting = indexesGroupByIsBaseOrNot - .getOrDefault(false, ImmutableList.of()).stream() - .filter(index -> preAggEnabledByHint(scan) - || checkPreAggStatus(scan, index.getId(), predicates, aggregateFunctions, groupingExprs).isOn()) - .collect(Collectors.toSet()); - // try to rewrite bitmap, hll by materialized index columns. - List candidatesWithRewriting = indexesGroupByIsBaseOrNot + Set candidatesWithRewriting = indexesGroupByIsBaseOrNot .getOrDefault(false, ImmutableList.of()).stream() - .filter(index -> !candidatesWithoutRewriting.contains(index)) .map(index -> rewriteAgg(index, scan, nonVirtualRequiredScanOutput, predicates, aggregateFunctions, groupingExprs)) .filter(aggRewriteResult -> checkPreAggStatus(scan, aggRewriteResult.index.getId(), predicates, // check pre-agg status of aggregate function that couldn't rewrite. aggFuncsDiff(aggregateFunctions, aggRewriteResult), groupingExprs).isOn()) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); + + Set candidatesWithoutRewriting = indexesGroupByIsBaseOrNot + .getOrDefault(false, ImmutableList.of()).stream() + .filter(index -> !candidatesWithRewriting.contains(index)) + .filter(index -> preAggEnabledByHint(scan) + || checkPreAggStatus(scan, index.getId(), predicates, aggregateFunctions, groupingExprs).isOn()) + .collect(Collectors.toSet()); List haveAllRequiredColumns = Streams.concat( candidatesWithoutRewriting.stream() @@ -1461,6 +1444,13 @@ private List replaceProjectList( .collect(Collectors.toList()); } + private List replaceRepeatOutput(LogicalRepeat repeat, + Map projectMap) { + return repeat.getOutputs().stream() + .map(expr -> (NamedExpression) ExpressionUtils.replaceNameExpression(expr, projectMap)) + .collect(Collectors.toList()); + } + private List nonVirtualGroupByExprs(LogicalAggregate agg) { return agg.getGroupByExpressions().stream() .filter(expr -> !(expr instanceof VirtualSlotReference)) diff --git a/regression-test/data/mv_p0/test_upper_alias/test_upper_alias.out b/regression-test/data/mv_p0/test_upper_alias/test_upper_alias.out new file mode 100644 index 00000000000000..e0c348fcf36bba --- /dev/null +++ b/regression-test/data/mv_p0/test_upper_alias/test_upper_alias.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv -- +XXX +YYY + +-- !select_mv -- +XXX +YYY + +-- !select_mv -- +wfsdf +wfsdf + diff --git a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy index f4a06ac04c340c..1552c5f67bee76 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_year/test_dup_mv_year.groovy @@ -57,7 +57,6 @@ suite ("test_dup_mv_year") { } sql "insert into d_table select 4,'2033-12-31','2033-12-31 01:02:03';" - Thread.sleep(1000) qt_select_star "select * from d_table order by k1;" diff --git a/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy new file mode 100644 index 00000000000000..301966baf0bf61 --- /dev/null +++ b/regression-test/suites/mv_p0/test_upper_alias/test_upper_alias.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite ("test_upper_alias") { + sql """set enable_nereids_planner=true""" + sql """SET enable_fallback_to_original_planner=false""" + sql """ drop table if exists test_0401;""" + + sql """ + CREATE TABLE test_0401 ( + `d_b` varchar(128) NULL, + `d_a` varchar(128) NULL, + `amt_b0` double NULL + ) ENGINE=OLAP + DUPLICATE KEY(`d_b`) + DISTRIBUTED BY HASH(`d_b`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """insert into test_0401 values('xxx', 'wfsdf', 9.30 );""" + + createMV (""" + create materialized view test_0401_mv as + select d_b, sum(amt_b0) as amt_b0 from test_0401 group by d_b; + """) + + createMV (""" + create materialized view test_0401_mv2 as + select d_a,d_b from test_0401; + """) + + sql """insert into test_0401 values('yyy', 'wfsdf', 91.310 );""" + + explain { + sql("SELECT upper(d_b) AS d_b FROM test_0401 GROUP BY upper(d_b) order by 1;") + contains "(test_0401_mv)" + } + qt_select_mv "SELECT upper(d_b) AS d_b FROM test_0401 GROUP BY upper(d_b) order by 1;" + + explain { + sql("SELECT upper(d_b) AS d_bb FROM test_0401 GROUP BY upper(d_b) order by 1;") + contains "(test_0401_mv)" + } + qt_select_mv "SELECT upper(d_b) AS d_bb FROM test_0401 GROUP BY upper(d_b) order by 1;" + + explain { + sql("SELECT d_a AS d_b FROM test_0401 order by 1;") + contains "(test_0401_mv2)" + } + qt_select_mv "SELECT d_a AS d_b FROM test_0401 order by 1;" +} From 41ecb51b2763b91c17b0a68b5a654b90c3185734 Mon Sep 17 00:00:00 2001 From: BiteTheDDDDt Date: Mon, 8 Apr 2024 17:04:11 +0800 Subject: [PATCH 4/4] remove --- .../mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out | 5 ----- .../ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy | 6 ------ 2 files changed, 11 deletions(-) diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out index f37d3ef54f31f6..c231b8737e40c5 100644 --- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out +++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.out @@ -18,11 +18,6 @@ 2 1 3 1 --- !select_mv -- -1 2 -2 2 -3 3 - -- !select_mv -- 1 2 diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy index 9b3e8ba371e328..148b23af7f9e84 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV1/testAggQueryOnAggMV1.groovy @@ -68,12 +68,6 @@ suite ("testAggQueryOnAggMV1") { } qt_select_mv "select deptno, count(deptno) from emps group by deptno order by deptno;" - explain { - sql("select deptno, sum(if(empid = 1, empid, salary)) from emps group by deptno;") - contains "(emps_mv_if)" - } - qt_select_mv "select deptno, sum(if(empid = 1, empid, salary)) from emps group by deptno order by deptno;" - explain { sql("select deptno, count(deptno) from emps where deptno=1 group by deptno order by deptno;") contains "(emps_mv_count_key)"