branch-3.0: [opt](mtmv) Support any_value aggregate function rollup rewrite when exsits aggregate materialized view #44412

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java

@@ -50,7 +50,6 @@
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.Repeat;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
@@ -155,7 +154,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
                             materializationContext.getShuttledExprToScanExprMapping(),
                             viewToQuerySlotMapping));
         }
-        return doRewriteQueryByView(queryStructInfo,
+        return aggregateRewriteByView(queryStructInfo,
                 viewToQuerySlotMapping,
                 queryTopPlanAndAggPair,
                 tempRewritedPlan,
@@ -167,7 +166,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
     /**
      * Aggregate function and group by expression rewrite impl
      */
-    protected LogicalAggregate<Plan> doRewriteQueryByView(
+    protected LogicalAggregate<Plan> aggregateRewriteByView(
             StructInfo queryStructInfo,
             SlotMapping viewToQuerySlotMapping,
             Pair<Plan, LogicalAggregate<Plan>> queryTopPlanAndAggPair,
@@ -426,12 +425,11 @@ private boolean isGroupByEquals(Pair<Plan, LogicalAggregate<Plan>> queryTopPlanA
         LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
         LogicalAggregate<Plan> viewAggregate = viewTopPlanAndAggPair.value();
 
-        Set<Expression> queryGroupShuttledExpression = new HashSet<>(ExpressionUtils.shuttleExpressionWithLineage(
+        Set<Expression> queryGroupByShuttledExpression = new HashSet<>(ExpressionUtils.shuttleExpressionWithLineage(
                 queryAggregate.getGroupByExpressions(), queryTopPlan, queryStructInfo.getTableBitSet()));
 
         // try to eliminate group by dimension by function dependency if group by expression is not in query
         Map<Expression, Expression> viewShuttledExpressionQueryBasedToGroupByExpressionMap = new HashMap<>();
-        Map<Expression, Expression> groupByExpressionToViewShuttledExpressionQueryBasedMap = new HashMap<>();
         List<Expression> viewGroupByExpressions = viewAggregate.getGroupByExpressions();
         List<? extends Expression> viewGroupByShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage(
                 viewGroupByExpressions, viewTopPlan, viewStructInfo.getTableBitSet());
@@ -443,43 +441,40 @@ private boolean isGroupByEquals(Pair<Plan, LogicalAggregate<Plan>> queryTopPlanA
                     viewToQuerySlotMapping.toSlotReferenceMap());
             viewShuttledExpressionQueryBasedToGroupByExpressionMap.put(viewGroupExpressionQueryBased,
                     viewExpression);
-            groupByExpressionToViewShuttledExpressionQueryBasedMap.put(viewExpression,
-                    viewGroupExpressionQueryBased
-            );
         }
-        if (queryGroupShuttledExpression.equals(viewShuttledExpressionQueryBasedToGroupByExpressionMap.keySet())) {
+        if (queryGroupByShuttledExpression.equals(viewShuttledExpressionQueryBasedToGroupByExpressionMap.keySet())) {
             // return true, if equals directly
             return true;
         }
-
-        boolean isGroupByEquals = false;
-        // check is equals by group by eliminate
-        isGroupByEquals |= isGroupByEqualsAfterGroupByEliminate(queryGroupShuttledExpression,
-                viewShuttledExpressionQueryBasedToGroupByExpressionMap,
-                groupByExpressionToViewShuttledExpressionQueryBasedMap,
-                viewAggregate,
-                cascadesContext);
-        // check is equals by equal filter eliminate
-        Optional<LogicalFilter<Plan>> filterOptional = tempRewrittenPlan.collectFirst(LogicalFilter.class::isInstance);
-        if (!filterOptional.isPresent()) {
-            return isGroupByEquals;
-        }
-        isGroupByEquals |= isGroupByEqualsAfterEqualFilterEliminate(
+        // Check is equals by equal filter eliminate
+        return isGroupByEqualsByFunctionDependency(
                 (LogicalPlan) tempRewrittenPlan,
-                queryGroupShuttledExpression,
+                queryGroupByShuttledExpression,
                 viewShuttledExpressionQueryBasedToGroupByExpressionMap,
                 materializationContext);
-        return isGroupByEquals;
     }
 
     /**
-     * Check group by is equals by equal filter eliminate
-     * For example query is select a, b, c from t1 where a = 1 and d = 'xx' group by a, b, c;
-     * mv is select a, b, c, d from t1 group by a, b, c, d;
-     * the group by expression between query and view is equals after equal filter eliminate
-     * should not aggregate roll up
+     * Check group by is equals by uniform function dependency
+     * For example query is:
+     * select
+     * a, b, c from t1
+     * where a = 1 and d = 'xx'
+     * group by a, b, c;
+     * mv is :
+     * select a, b, c, d
+     * from t1
+     * group by a, b, c, d;
+     * After group by key eliminate, the query group by is b, c
+     * but mv is group by a, b, c, d, the group by a and d of mv is more dimensions than the query
+     * But in tempRewrittenPlan is as following:
+     * select *
+     * from mv
+     * where a = 1 and d = 'xx'
+     * We can get group by a and d is uniform by function dependency info,
+     * so the group by expression between query and view is equals, should not aggregate roll up
      * */
-    private static boolean isGroupByEqualsAfterEqualFilterEliminate(
+    private static boolean isGroupByEqualsByFunctionDependency(
             LogicalPlan tempRewrittenPlan,
             Set<Expression> queryGroupShuttledExpression,
             Map<Expression, Expression> viewShuttledExprQueryBasedToViewGroupByExprMap,
@@ -541,50 +536,50 @@ private static boolean isGroupByEqualsAfterEqualFilterEliminate(
      * viewAggregate
      * 4. check the viewAggregate group by expression is equals queryAggregate expression or not
      */
-    private static boolean isGroupByEqualsAfterGroupByEliminate(Set<Expression> queryGroupShuttledExpression,
+    private static boolean isGroupByEqualsAfterGroupByEliminate(Set<Expression> queryGroupByShuttledExpression,
             Map<Expression, Expression> viewShuttledExpressionQueryBasedToGroupByExpressionMap,
             Map<Expression, Expression> groupByExpressionToViewShuttledExpressionQueryBasedMap,
             LogicalAggregate<Plan> viewAggregate,
             CascadesContext cascadesContext) {
-        List<NamedExpression> projects = new ArrayList<>();
-        // construct projects query used by view group expressions
-        for (Expression expression : queryGroupShuttledExpression) {
+        List<NamedExpression> viewProjects = new ArrayList<>();
+        // construct viewProjects query used by view group expressions
+        for (Expression expression : queryGroupByShuttledExpression) {
             Expression chosenExpression = viewShuttledExpressionQueryBasedToGroupByExpressionMap.get(expression);
             if (chosenExpression == null) {
                 return false;
             }
-            projects.add(chosenExpression instanceof NamedExpression
+            viewProjects.add(chosenExpression instanceof NamedExpression
                     ? (NamedExpression) chosenExpression : new Alias(chosenExpression));
         }
-        LogicalProject<LogicalAggregate<Plan>> project = new LogicalProject<>(projects, viewAggregate);
+        LogicalProject<LogicalAggregate<Plan>> viewProject = new LogicalProject<>(viewProjects, viewAggregate);
         // try to eliminate view group by expression which is not in query group by expression
         Plan rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext,
                 childContext -> {
                     Rewriter.getCteChildrenRewriter(childContext,
                             ImmutableList.of(Rewriter.topDown(new EliminateGroupByKey()))).execute();
                     return childContext.getRewritePlan();
-                }, project, project);
+                }, viewProject, viewProject);
 
-        Optional<LogicalAggregate<Plan>> aggreagateOptional =
+        Optional<LogicalAggregate<Plan>> viewAggreagateOptional =
                 rewrittenPlan.collectFirst(LogicalAggregate.class::isInstance);
-        if (!aggreagateOptional.isPresent()) {
+        if (!viewAggreagateOptional.isPresent()) {
             return false;
         }
         // check result after view group by eliminate by functional dependency
-        List<Expression> viewEliminatedGroupByExpressions = aggreagateOptional.get().getGroupByExpressions();
-        if (viewEliminatedGroupByExpressions.size() != queryGroupShuttledExpression.size()) {
+        List<Expression> viewEliminatedGroupByExpressions = viewAggreagateOptional.get().getGroupByExpressions();
+        if (viewEliminatedGroupByExpressions.size() != queryGroupByShuttledExpression.size()) {
             return false;
         }
         Set<Expression> viewGroupShuttledExpressionQueryBased = new HashSet<>();
-        for (Expression viewExpression : aggreagateOptional.get().getGroupByExpressions()) {
+        for (Expression viewExpression : viewAggreagateOptional.get().getGroupByExpressions()) {
             Expression viewExpressionQueryBased =
                     groupByExpressionToViewShuttledExpressionQueryBasedMap.get(viewExpression);
             if (viewExpressionQueryBased == null) {
                 return false;
             }
             viewGroupShuttledExpressionQueryBased.add(viewExpressionQueryBased);
         }
-        return queryGroupShuttledExpression.equals(viewGroupShuttledExpressionQueryBased);
+        return queryGroupByShuttledExpression.equals(viewGroupShuttledExpressionQueryBased);
     }
 
     /**

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java

@@ -124,7 +124,7 @@ protected Plan rewriteQueryByView(MatchMode matchMode, StructInfo queryStructInf
                     () -> String.format("query aggregate = %s", queryAggregate.treeString()));
             return null;
         }
-        return doRewriteQueryByView(queryStructInfo,
+        return aggregateRewriteByView(queryStructInfo,
                 viewToQuerySlotMapping,
                 queryTopPlanAndAggPair,
                 tempRewritedPlan,

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/AggFunctionRollUpHandler.java

@@ -62,12 +62,12 @@ public abstract Function doRollup(
     /**
      * Extract the function arguments by functionWithAny pattern
      * Such as functionWithAny def is bitmap_union(to_bitmap(Any.INSTANCE)),
-     * actualFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end))
+     * actualExpression is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end))
      * after extracting, the return argument is: case when a = 5 then 1 else 2 end
      */
-    protected static List<Expression> extractArguments(Expression functionWithAny, Function actualFunction) {
+    protected static List<Expression> extractArguments(Expression functionWithAny, Expression actualExpression) {
         Set<Object> exprSetToRemove = functionWithAny.collectToSet(expr -> !(expr instanceof Any));
-        return actualFunction.collectFirst(expr ->
+        return actualExpression.collectFirst(expr ->
                         exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr)))
                 .map(expr -> ImmutableList.of((Expression) expr)).orElse(ImmutableList.of());
     }

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/ContainDistinctFunctionRollupHandler.java

@@ -76,6 +76,7 @@ public boolean canRollup(AggregateFunction queryAggregateFunction,
         Set<Expression> mvExpressionsQueryBased = mvExprToMvScanExprQueryBased.keySet();
         Set<Slot> aggregateFunctionParamSlots = queryAggregateFunctionShuttled.collectToSet(Slot.class::isInstance);
         if (aggregateFunctionParamSlots.stream().anyMatch(slot -> !mvExpressionsQueryBased.contains(slot))) {
+            // If query use any slot not in view, can not roll up
             return false;
         }
         return true;

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/DirectRollupHandler.java

@@ -52,9 +52,7 @@ public boolean canRollup(
                 && viewExpression instanceof NullableAggregateFunction
                 ? ((NullableAggregateFunction) queryAggregateFunctionShuttled).equalsIgnoreNullable(viewExpression)
                 : queryAggregateFunctionShuttled.equals(viewExpression);
-        return isEquals && MappingRollupHandler.AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.keySet().stream()
-                .noneMatch(aggFunction -> aggFunction.equals(queryAggregateFunction))
-                && !(queryAggregateFunction instanceof Combinator);
+        return isEquals && !(queryAggregateFunction instanceof Combinator);
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/rollup/MappingRollupHandler.java

@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue;
 import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion;
 import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
@@ -132,6 +133,9 @@ public class MappingRollupHandler extends AggFunctionRollUpHandler {
         AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new HllUnion(Any.INSTANCE),
                 new HllUnion(Any.INSTANCE));
 
+        // support roll up when any_value
+        AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new AnyValue(Any.INSTANCE),
+                Any.INSTANCE);
     }
 
     @Override
@@ -141,24 +145,20 @@ public boolean canRollup(AggregateFunction queryAggregateFunction,
             Map<Expression, Expression> mvExprToMvScanExprQueryBasedMap) {
         // handle complex functions roll up by mapping and combinator expression
         // eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param))
-        Expression viewExpression = mvExprToMvScanExprQueryBasedPair.key();
-        if (!super.canRollup(queryAggregateFunction, queryAggregateFunctionShuttled,
-                mvExprToMvScanExprQueryBasedPair, mvExprToMvScanExprQueryBasedMap)) {
-            return false;
-        }
-        Function viewFunction = (Function) viewExpression;
-        for (Map.Entry<Function, Collection<Expression>> equivalentFunctionEntry :
+        Expression actualViewExpression = mvExprToMvScanExprQueryBasedPair.key();
+        for (Map.Entry<Function, Collection<Expression>> queryToViewEquivalentMapEntry :
                 AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.asMap().entrySet()) {
-            if (equivalentFunctionEntry.getKey().equals(queryAggregateFunction)) {
-                // check is have equivalent function or not
-                for (Expression equivalentFunction : equivalentFunctionEntry.getValue()) {
-                    if (!Any.equals(equivalentFunction, viewFunction)) {
+            if (queryToViewEquivalentMapEntry.getKey().equals(queryAggregateFunction)) {
+                for (Expression mappedViewExpression : queryToViewEquivalentMapEntry.getValue()) {
+                    if (!Any.equals(mappedViewExpression, actualViewExpression)) {
+                        // check the mapping view expression is equivalent with actual view expression
                         continue;
                     }
                     // check param in query function is same as the view function
-                    List<Expression> viewFunctionArguments = extractArguments(equivalentFunction, viewFunction);
-                    List<Expression> queryFunctionArguments =
-                            extractArguments(equivalentFunctionEntry.getKey(), queryAggregateFunction);
+                    List<Expression> viewFunctionArguments = extractArguments(mappedViewExpression,
+                            actualViewExpression);
+                    List<Expression> queryFunctionArguments = extractArguments(queryToViewEquivalentMapEntry.getKey(),
+                            queryAggregateFunction);
                     // check argument size,we only support roll up function which has only one argument currently
                     if (queryFunctionArguments.size() != 1 || viewFunctionArguments.size() != 1) {
                         continue;

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AnyValue.java

@@ -20,6 +20,7 @@
 import org.apache.doris.catalog.FunctionSignature;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.coercion.AnyDataType;
@@ -33,7 +34,7 @@
  * AggregateFunction 'any_value'. This class is generated by GenerateFunction.
  */
 public class AnyValue extends NullableAggregateFunction
-        implements UnaryExpression, ExplicitlyCastableSignature {
+        implements UnaryExpression, ExplicitlyCastableSignature, RollUpTrait {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
             FunctionSignature.retArgType(0).args(AnyDataType.INSTANCE_WITHOUT_INDEX)
@@ -80,4 +81,14 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
     public List<FunctionSignature> getSignatures() {
         return SIGNATURES;
     }
+
+    @Override
+    public Function constructRollUp(Expression param, Expression... varParams) {
+        return new AnyValue(this.distinct, this.alwaysNullable, param);
+    }
+
+    @Override
+    public boolean canRollUp() {
+        return true;
+    }
 }

regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/any_value_roll_up.out

@@ -0,0 +1,73 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !query1_0_before --
+3	o
+3	o
+3	o
+3	o
+4	o
+
+-- !query1_0_after --
+3	o
+3	o
+3	o
+3	o
+4	o
+
+-- !query2_0_before --
+3	o
+3	o
+3	o
+3	o
+4	o
+
+-- !query2_0_after --
+3	o
+3	o
+3	o
+3	o
+4	o
+
+-- !query3_0_before --
+3	3	o
+
+-- !query3_0_after --
+3	3	o
+
+-- !query4_0_before --
+3	o
+
+-- !query4_0_after --
+3	o
+
+-- !query5_0_before --
+3	3	o
+
+-- !query5_0_after --
+3	3	o
+
+-- !query5_1_before --
+3	3	o
+
+-- !query5_1_after --
+3	3	o
+
+-- !query6_0_before --
+3	o
+
+-- !query6_0_after --
+3	o
+
+-- !query6_1_before --
+3	o
+3	o
+3	o
+3	o
+4	o
+
+-- !query6_1_after --
+3	o
+3	o
+3	o
+3	o
+4	o
+