diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index a91ab2ea5ba434..45a2df77758f13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1577,6 +1577,11 @@ private void transferToMaster() { SessionVariable.NEREIDS_TIMEOUT_SECOND, "30"); } } + if (journalVersion <= FeMetaVersion.VERSION_133) { + VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1", + SessionVariable.ENABLE_MATERIALIZED_VIEW_REWRITE, + "true"); + } } getPolicyMgr().createDefaultStoragePolicy(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java index 2d39852dd1861a..bff2f0ba0ddabd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java @@ -88,16 +88,23 @@ public Rule build() { return new LogicalAggregate<>(repeat.getGroupByExpressions(), repeat.getOutputExpressions(), repeat.child()); } - checkRepeatLegality(repeat); - repeat = removeDuplicateColumns(repeat); - // add virtual slot, LogicalAggregate and LogicalProject for normalize - LogicalAggregate agg = normalizeRepeat(repeat); - return dealSlotAppearBothInAggFuncAndGroupingSets(agg); + return doNormalize(repeat); }) ); } - private LogicalRepeat removeDuplicateColumns(LogicalRepeat repeat) { + /** + * Normalize repeat, this can be used directly, if optimize the repeat + */ + public static LogicalAggregate doNormalize(LogicalRepeat repeat) { + checkRepeatLegality(repeat); + repeat = removeDuplicateColumns(repeat); + // add virtual slot, LogicalAggregate and LogicalProject for normalize + LogicalAggregate agg = normalizeRepeat(repeat); + return dealSlotAppearBothInAggFuncAndGroupingSets(agg); + } + + private static LogicalRepeat removeDuplicateColumns(LogicalRepeat repeat) { List> groupingSets = repeat.getGroupingSets(); ImmutableList.Builder> builder = ImmutableList.builder(); for (List sets : groupingSets) { @@ -107,11 +114,11 @@ private LogicalRepeat removeDuplicateColumns(LogicalRepeat repeat) { return repeat.withGroupSets(builder.build()); } - private void checkRepeatLegality(LogicalRepeat repeat) { + private static void checkRepeatLegality(LogicalRepeat repeat) { checkGroupingSetsSize(repeat); } - private void checkGroupingSetsSize(LogicalRepeat repeat) { + private static void checkGroupingSetsSize(LogicalRepeat repeat) { Set flattenGroupingSetExpr = ImmutableSet.copyOf( ExpressionUtils.flatExpressions(repeat.getGroupingSets())); if (flattenGroupingSetExpr.size() > LogicalRepeat.MAX_GROUPING_SETS_NUM) { @@ -121,7 +128,7 @@ private void checkGroupingSetsSize(LogicalRepeat repeat) { } } - private LogicalAggregate normalizeRepeat(LogicalRepeat repeat) { + private static LogicalAggregate normalizeRepeat(LogicalRepeat repeat) { Set needToSlotsGroupingExpr = collectNeedToSlotGroupingExpr(repeat); NormalizeToSlotContext groupingExprContext = buildContext(repeat, needToSlotsGroupingExpr); Map groupingExprMap = groupingExprContext.getNormalizeToSlotMap(); @@ -195,14 +202,14 @@ private LogicalAggregate normalizeRepeat(LogicalRepeat repeat) { Optional.of(normalizedRepeat), normalizedRepeat); } - private Set collectNeedToSlotGroupingExpr(LogicalRepeat repeat) { + private static Set collectNeedToSlotGroupingExpr(LogicalRepeat repeat) { // grouping sets should be pushed down, e.g. grouping sets((k + 1)), // we should push down the `k + 1` to the bottom plan return ImmutableSet.copyOf( ExpressionUtils.flatExpressions(repeat.getGroupingSets())); } - private Set collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(LogicalRepeat repeat) { + private static Set collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(LogicalRepeat repeat) { Set groupingScalarFunctions = ExpressionUtils.collect( repeat.getOutputExpressions(), GroupingScalarFunction.class::isInstance); ImmutableSet.Builder argumentsSetBuilder = ImmutableSet.builder(); @@ -234,7 +241,7 @@ private Set collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(Logi .build(); } - private Plan pushDownProject(Set pushedExprs, Plan originBottomPlan) { + private static Plan pushDownProject(Set pushedExprs, Plan originBottomPlan) { if (!pushedExprs.equals(originBottomPlan.getOutputSet()) && !pushedExprs.isEmpty()) { return new LogicalProject<>(ImmutableList.copyOf(pushedExprs), originBottomPlan); } @@ -242,7 +249,7 @@ private Plan pushDownProject(Set pushedExprs, Plan originBottom } /** buildContext */ - public NormalizeToSlotContext buildContext(Repeat repeat, + public static NormalizeToSlotContext buildContext(Repeat repeat, Set sourceExpressions) { Set aliases = ExpressionUtils.collect(repeat.getOutputExpressions(), Alias.class::isInstance); Map existsAliasMap = Maps.newLinkedHashMap(); @@ -268,7 +275,7 @@ public NormalizeToSlotContext buildContext(Repeat repeat, return new NormalizeToSlotContext(normalizeToSlotMap); } - private Optional toGroupingSetExpressionPushDownTriplet( + private static Optional toGroupingSetExpressionPushDownTriplet( Expression expression, @Nullable Alias existsAlias) { NormalizeToSlotTriplet originTriplet = NormalizeToSlotTriplet.toTriplet(expression, existsAlias); SlotReference remainSlot = (SlotReference) originTriplet.remainExpr; @@ -276,7 +283,8 @@ private Optional toGroupingSetExpressionPushDownTriplet( return Optional.of(new NormalizeToSlotTriplet(expression, newSlot, originTriplet.pushedExpr)); } - private Expression normalizeAggFuncChildrenAndGroupingScalarFunc(NormalizeToSlotContext context, Expression expr) { + private static Expression normalizeAggFuncChildrenAndGroupingScalarFunc(NormalizeToSlotContext context, + Expression expr) { if (expr instanceof AggregateFunction) { AggregateFunction function = (AggregateFunction) expr; List normalizedRealExpressions = context.normalizeToUseSlotRef(function.getArguments()); @@ -293,7 +301,7 @@ private Expression normalizeAggFuncChildrenAndGroupingScalarFunc(NormalizeToSlot } } - private Set getExistsAlias(LogicalRepeat repeat, + private static Set getExistsAlias(LogicalRepeat repeat, Map groupingExprMap) { Set existsAlias = Sets.newHashSet(); Set aliases = ExpressionUtils.collect(repeat.getOutputExpressions(), Alias.class::isInstance); @@ -320,7 +328,7 @@ private Set getExistsAlias(LogicalRepeat repeat, * +--LogicalRepeat (groupingSets=[[a#0]], outputExpr=[a#0, a#3, GROUPING_ID#1] * +--LogicalProject (projects =[a#0, a#0 as `a`#3]) */ - private LogicalAggregate dealSlotAppearBothInAggFuncAndGroupingSets( + private static LogicalAggregate dealSlotAppearBothInAggFuncAndGroupingSets( @NotNull LogicalAggregate aggregate) { LogicalRepeat repeat = (LogicalRepeat) aggregate.child(); Map commonSlotToAliasMap = getCommonSlotToAliasMap(repeat, aggregate); @@ -367,7 +375,8 @@ private LogicalAggregate dealSlotAppearBothInAggFuncAndGroupingSets( return aggregate.withAggOutput(newOutputExpressions); } - private Map getCommonSlotToAliasMap(LogicalRepeat repeat, LogicalAggregate aggregate) { + private static Map getCommonSlotToAliasMap(LogicalRepeat repeat, + LogicalAggregate aggregate) { List aggregateFunctions = CollectNonWindowedAggFuncs.collect(aggregate.getOutputExpressions()); ImmutableSet.Builder aggUsedSlotBuilder = ImmutableSet.builder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java index 6883247ce16196..ee5477747bd1ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.analysis.NormalizeRepeat; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanCheckContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PlanSplitContext; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; @@ -32,13 +33,16 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; -import org.apache.doris.nereids.trees.expressions.functions.agg.RollUpTrait; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Repeat; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; import org.apache.doris.nereids.util.ExpressionUtils; @@ -51,7 +55,9 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -91,10 +97,16 @@ protected Plan rewriteQueryByView(MatchMode matchMode, () -> String.format("query plan = %s\n", queryStructInfo.getOriginalPlan().treeString())); return null; } - // Firstly,if group by expression between query and view is equals, try to rewrite expression directly Plan queryTopPlan = queryTopPlanAndAggPair.key(); - if (isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, viewToQuerySlotMapping, queryStructInfo, - viewStructInfo, materializationContext)) { + LogicalAggregate queryAggregate = queryTopPlanAndAggPair.value(); + if (!checkCompatibility(queryStructInfo, queryAggregate, viewTopPlanAndAggPair.value(), + materializationContext)) { + return null; + } + boolean queryContainsGroupSets = queryAggregate.getSourceRepeat().isPresent(); + // If group by expression between query and view is equals, try to rewrite expression directly + if (!queryContainsGroupSets && isGroupByEquals(queryTopPlanAndAggPair, viewTopPlanAndAggPair, + viewToQuerySlotMapping, queryStructInfo, viewStructInfo, materializationContext)) { List rewrittenQueryExpressions = rewriteExpression(queryTopPlan.getOutput(), queryTopPlan, materializationContext.getShuttledExprToScanExprMapping(), @@ -124,20 +136,6 @@ protected Plan rewriteQueryByView(MatchMode matchMode, materializationContext.getShuttledExprToScanExprMapping(), viewToQuerySlotMapping)); } - // if view is scalar aggregate but query is not. Or if query is scalar aggregate but view is not - // Should not rewrite - List queryGroupByExpressions = queryTopPlanAndAggPair.value().getGroupByExpressions(); - List viewGroupByExpressions = viewTopPlanAndAggPair.value().getGroupByExpressions(); - if ((queryGroupByExpressions.isEmpty() && !viewGroupByExpressions.isEmpty()) - || (!queryGroupByExpressions.isEmpty() && viewGroupByExpressions.isEmpty())) { - materializationContext.recordFailReason(queryStructInfo, - "only one the of query or view is scalar aggregate and " - + "can not rewrite expression meanwhile", - () -> String.format("query aggregate = %s,\n view aggregate = %s,\n", - queryTopPlanAndAggPair.value().treeString(), - viewTopPlanAndAggPair.value().treeString())); - return null; - } // try to roll up. // split the query top plan expressions to group expressions and functions, if can not, bail out. Pair, Set> queryGroupAndFunctionPair @@ -147,46 +145,31 @@ protected Plan rewriteQueryByView(MatchMode matchMode, // try to rewrite, contains both roll up aggregate functions and aggregate group expression List finalOutputExpressions = new ArrayList<>(); List finalGroupExpressions = new ArrayList<>(); - List queryExpressions = queryTopPlan.getOutput(); // permute the mv expr mapping to query based Map mvExprToMvScanExprQueryBased = materializationContext.getShuttledExprToScanExprMapping().keyPermute(viewToQuerySlotMapping) .flattenMap().get(0); - for (Expression topExpression : queryExpressions) { - // if agg function, try to roll up and rewrite + for (Expression topExpression : queryTopPlan.getOutput()) { if (queryTopPlanFunctionSet.contains(topExpression)) { - Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage( - topExpression, - queryTopPlan, - queryStructInfo.getTableBitSet()); - AggregateExpressionRewriteContext context = new AggregateExpressionRewriteContext( - false, mvExprToMvScanExprQueryBased, queryTopPlan, queryStructInfo.getTableBitSet()); - // queryFunctionShuttled maybe sum(column) + count(*), so need to use expression rewriter - Expression rollupedExpression = queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER, - context); - if (!context.isValid()) { - materializationContext.recordFailReason(queryStructInfo, - "Query function roll up fail", - () -> String.format("queryFunctionShuttled = %s,\n mvExprToMvScanExprQueryBased = %s", - queryFunctionShuttled, mvExprToMvScanExprQueryBased)); + // if agg function, try to roll up and rewrite + Expression rollupedExpression = tryRewriteExpression(queryStructInfo, topExpression, + mvExprToMvScanExprQueryBased, false, materializationContext, + "Query function roll up fail", + () -> String.format("queryExpression = %s,\n mvExprToMvScanExprQueryBased = %s", + topExpression, mvExprToMvScanExprQueryBased)); + if (rollupedExpression == null) { return null; } finalOutputExpressions.add(new Alias(rollupedExpression)); } else { - // if group by expression, try to rewrite group by expression - Expression queryGroupShuttledExpr = ExpressionUtils.shuttleExpressionWithLineage( - topExpression, queryTopPlan, queryStructInfo.getTableBitSet()); - AggregateExpressionRewriteContext context = new AggregateExpressionRewriteContext(true, - mvExprToMvScanExprQueryBased, queryTopPlan, queryStructInfo.getTableBitSet()); - // group by expression maybe group by a + b, so we need expression rewriter - Expression rewrittenGroupByExpression = queryGroupShuttledExpr.accept(AGGREGATE_EXPRESSION_REWRITER, - context); - if (!context.isValid()) { + // if group by dimension, try to rewrite + Expression rewrittenGroupByExpression = tryRewriteExpression(queryStructInfo, topExpression, + mvExprToMvScanExprQueryBased, true, materializationContext, + "View dimensions doesn't not cover the query dimensions", + () -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryExpression is %s", + mvExprToMvScanExprQueryBased, topExpression)); + if (rewrittenGroupByExpression == null) { // group expr can not rewrite by view - materializationContext.recordFailReason(queryStructInfo, - "View dimensions doesn't not cover the query dimensions", - () -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryGroupShuttledExpr is %s", - mvExprToMvScanExprQueryBased, queryGroupShuttledExpr)); return null; } NamedExpression groupByExpression = rewrittenGroupByExpression instanceof NamedExpression @@ -195,26 +178,19 @@ protected Plan rewriteQueryByView(MatchMode matchMode, finalGroupExpressions.add(groupByExpression); } } - // add project to guarantee group by column ref is slot reference, - // this is necessary because physical createHash will need slotReference later + List queryGroupByExpressions = queryAggregate.getGroupByExpressions(); + // handle the scene that query top plan not use the group by in query bottom aggregate if (queryGroupByExpressions.size() != queryTopPlanGroupBySet.size()) { for (Expression expression : queryGroupByExpressions) { if (queryTopPlanGroupBySet.contains(expression)) { continue; } - Expression queryGroupShuttledExpr = ExpressionUtils.shuttleExpressionWithLineage( - expression, queryTopPlan, queryStructInfo.getTableBitSet()); - AggregateExpressionRewriteContext context = new AggregateExpressionRewriteContext(true, - mvExprToMvScanExprQueryBased, queryTopPlan, queryStructInfo.getTableBitSet()); - // group by expression maybe group by a + b, so we need expression rewriter - Expression rewrittenGroupByExpression = queryGroupShuttledExpr.accept(AGGREGATE_EXPRESSION_REWRITER, - context); - if (!context.isValid()) { - // group expr can not rewrite by view - materializationContext.recordFailReason(queryStructInfo, - "View dimensions doesn't not cover the query dimensions in bottom agg ", - () -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryGroupShuttledExpr is %s", - mvExprToMvScanExprQueryBased, queryGroupShuttledExpr)); + Expression rewrittenGroupByExpression = tryRewriteExpression(queryStructInfo, expression, + mvExprToMvScanExprQueryBased, true, materializationContext, + "View dimensions doesn't not cover the query dimensions in bottom agg ", + () -> String.format("mvExprToMvScanExprQueryBased is %s,\n expression is %s", + mvExprToMvScanExprQueryBased, expression)); + if (rewrittenGroupByExpression == null) { return null; } NamedExpression groupByExpression = rewrittenGroupByExpression instanceof NamedExpression @@ -222,31 +198,96 @@ protected Plan rewriteQueryByView(MatchMode matchMode, finalGroupExpressions.add(groupByExpression); } } - List copiedFinalGroupExpressions = new ArrayList<>(finalGroupExpressions); - List projectsUnderAggregate = copiedFinalGroupExpressions.stream() - .map(NamedExpression.class::cast) - .collect(Collectors.toList()); - projectsUnderAggregate.addAll(tempRewritedPlan.getOutput()); - LogicalProject mvProject = new LogicalProject<>(projectsUnderAggregate, tempRewritedPlan); - // add agg rewrite - Map projectOutPutExprIdMap = mvProject.getOutput().stream() - .distinct() - .collect(Collectors.toMap(NamedExpression::getExprId, slot -> slot)); - // make the expressions to re reference project output - finalGroupExpressions = finalGroupExpressions.stream() - .map(expr -> { - ExprId exprId = ((NamedExpression) expr).getExprId(); - if (projectOutPutExprIdMap.containsKey(exprId)) { - return projectOutPutExprIdMap.get(exprId); + if (queryContainsGroupSets) { + // construct group sets for repeat + List> rewrittenGroupSetsExpressions = new ArrayList<>(); + List> groupingSets = queryAggregate.getSourceRepeat().get().getGroupingSets(); + for (List groupingSet : groupingSets) { + if (groupingSet.isEmpty()) { + rewrittenGroupSetsExpressions.add(ImmutableList.of()); + } else { + List rewrittenGroupSetExpressions = new ArrayList<>(); + for (Expression expression : groupingSet) { + Expression rewrittenGroupByExpression = tryRewriteExpression(queryStructInfo, expression, + mvExprToMvScanExprQueryBased, true, materializationContext, + "View dimensions doesn't not cover the query group set dimensions", + () -> String.format("mvExprToMvScanExprQueryBased is %s,\n queryExpression is %s", + mvExprToMvScanExprQueryBased, expression)); + if (rewrittenGroupByExpression == null) { + return null; + } + rewrittenGroupSetExpressions.add(rewrittenGroupByExpression); } - return (NamedExpression) expr; - }) - .collect(Collectors.toList()); - finalOutputExpressions = finalOutputExpressions.stream() - .map(expr -> projectOutPutExprIdMap.containsKey(expr.getExprId()) - ? projectOutPutExprIdMap.get(expr.getExprId()) : expr) - .collect(Collectors.toList()); - return new LogicalAggregate(finalGroupExpressions, finalOutputExpressions, mvProject); + rewrittenGroupSetsExpressions.add(rewrittenGroupSetExpressions); + } + } + LogicalRepeat repeat = new LogicalRepeat<>(rewrittenGroupSetsExpressions, + finalOutputExpressions, tempRewritedPlan); + return NormalizeRepeat.doNormalize(repeat); + } + return new LogicalAggregate<>(finalGroupExpressions, finalOutputExpressions, tempRewritedPlan); + } + + /** + * Try to rewrite query expression by view, contains both group by dimension and aggregate function + */ + protected Expression tryRewriteExpression(StructInfo queryStructInfo, Expression queryExpression, + Map mvShuttledExprToMvScanExprQueryBased, boolean isGroupBy, + MaterializationContext materializationContext, String summaryIfFail, Supplier detailIfFail) { + Expression queryFunctionShuttled = ExpressionUtils.shuttleExpressionWithLineage( + queryExpression, + queryStructInfo.getTopPlan(), + queryStructInfo.getTableBitSet()); + AggregateExpressionRewriteContext expressionRewriteContext = new AggregateExpressionRewriteContext( + isGroupBy, mvShuttledExprToMvScanExprQueryBased, queryStructInfo.getTopPlan(), + queryStructInfo.getTableBitSet()); + Expression rewrittenExpression = queryFunctionShuttled.accept(AGGREGATE_EXPRESSION_REWRITER, + expressionRewriteContext); + if (!expressionRewriteContext.isValid()) { + materializationContext.recordFailReason(queryStructInfo, summaryIfFail, detailIfFail); + return null; + } + return rewrittenExpression; + } + + /** + * Check query and view aggregate compatibility + */ + private static boolean checkCompatibility( + StructInfo queryStructInfo, + LogicalAggregate queryAggregate, LogicalAggregate viewAggregate, + MaterializationContext materializationContext) { + // if view is scalar aggregate but query is not. Or if query is scalar aggregate but view is not + // Should not rewrite + List queryGroupByExpressions = queryAggregate.getGroupByExpressions(); + List viewGroupByExpressions = viewAggregate.getGroupByExpressions(); + if (!queryGroupByExpressions.isEmpty() && viewGroupByExpressions.isEmpty()) { + materializationContext.recordFailReason(queryStructInfo, + "only one the of query or view is scalar aggregate and " + + "can not rewrite expression meanwhile", + () -> String.format("query aggregate = %s,\n view aggregate = %s,\n", + queryAggregate.treeString(), + viewAggregate.treeString())); + return false; + } + boolean queryHasGroupSets = queryAggregate.getSourceRepeat() + .map(repeat -> repeat.getGroupingSets().size()).orElse(0) > 0; + boolean viewHasGroupSets = viewAggregate.getSourceRepeat() + .map(repeat -> repeat.getGroupingSets().size()).orElse(0) > 0; + // if both query and view has no group sets, maybe rewrite + if (!queryHasGroupSets && !viewHasGroupSets) { + return true; + } + // if both query and view has group sets, or query doesn't hava, mv have, not supported + if ((queryHasGroupSets && viewHasGroupSets) || (!queryHasGroupSets && viewHasGroupSets)) { + materializationContext.recordFailReason(queryStructInfo, + "both query and view have group sets, or query doesn't have but view have, not supported", + () -> String.format("query aggregate = %s,\n view aggregate = %s,\n", + queryAggregate.treeString(), + viewAggregate.treeString())); + return false; + } + return true; } private boolean isGroupByEquals(Pair> queryTopPlanAndAggPair, @@ -259,14 +300,18 @@ private boolean isGroupByEquals(Pair> queryTopPlanA Plan viewTopPlan = viewTopPlanAndAggPair.key(); LogicalAggregate queryAggregate = queryTopPlanAndAggPair.value(); LogicalAggregate viewAggregate = viewTopPlanAndAggPair.value(); - Set queryGroupShuttledExpression = new HashSet<>( - ExpressionUtils.shuttleExpressionWithLineage( - queryAggregate.getGroupByExpressions(), queryTopPlan, queryStructInfo.getTableBitSet())); - Set viewGroupShuttledExpressionQueryBased = ExpressionUtils.shuttleExpressionWithLineage( - viewAggregate.getGroupByExpressions(), viewTopPlan, viewStructInfo.getTableBitSet()) - .stream() - .map(expr -> ExpressionUtils.replace(expr, viewToQuerySlotMapping.toSlotReferenceMap())) - .collect(Collectors.toSet()); + + Set queryGroupShuttledExpression = new HashSet<>(); + for (Expression queryExpression : ExpressionUtils.shuttleExpressionWithLineage( + queryAggregate.getGroupByExpressions(), queryTopPlan, queryStructInfo.getTableBitSet())) { + queryGroupShuttledExpression.add(queryExpression); + } + Set viewGroupShuttledExpressionQueryBased = new HashSet<>(); + for (Expression viewExpression : ExpressionUtils.shuttleExpressionWithLineage( + viewAggregate.getGroupByExpressions(), viewTopPlan, viewStructInfo.getTableBitSet())) { + viewGroupShuttledExpressionQueryBased.add( + ExpressionUtils.replace(viewExpression, viewToQuerySlotMapping.toSlotReferenceMap())); + } return queryGroupShuttledExpression.equals(viewGroupShuttledExpressionQueryBased); } @@ -305,23 +350,6 @@ private static Function rollup(AggregateFunction queryAggregateFunction, return null; } - // Check the aggregate function can roll up or not, return true if could roll up - // if view aggregate function is distinct or is in the un supported rollup functions, it doesn't support - // roll up. - private static boolean canRollup(Expression rollupExpression) { - if (rollupExpression == null) { - return false; - } - if (rollupExpression instanceof Function && !(rollupExpression instanceof AggregateFunction)) { - return false; - } - if (rollupExpression instanceof AggregateFunction) { - AggregateFunction aggregateFunction = (AggregateFunction) rollupExpression; - return !aggregateFunction.isDistinct() && aggregateFunction instanceof RollUpTrait; - } - return true; - } - private Pair, Set> topPlanSplitToGroupAndFunction( Pair> topPlanAndAggPair, StructInfo queryStructInfo) { LogicalAggregate bottomQueryAggregate = topPlanAndAggPair.value(); @@ -405,11 +433,40 @@ public Expression visitAggregateFunction(AggregateFunction aggregateFunction, return rollupAggregateFunction; } + @Override + public Expression visitGroupingScalarFunction(GroupingScalarFunction groupingScalarFunction, + AggregateExpressionRewriteContext context) { + List children = groupingScalarFunction.children(); + List rewrittenChildren = new ArrayList<>(); + for (Expression child : children) { + Expression rewrittenChild = child.accept(this, context); + if (!context.isValid()) { + return groupingScalarFunction; + } + rewrittenChildren.add(rewrittenChild); + } + return groupingScalarFunction.withChildren(rewrittenChildren); + } + @Override public Expression visitSlot(Slot slot, AggregateExpressionRewriteContext rewriteContext) { if (!rewriteContext.isValid()) { return slot; } + if (slot instanceof VirtualSlotReference) { + Optional originExpression = ((VirtualSlotReference) slot).getOriginExpression(); + if (!originExpression.isPresent()) { + return Repeat.generateVirtualGroupingIdSlot(); + } else { + GroupingScalarFunction groupingScalarFunction = originExpression.get(); + groupingScalarFunction = + (GroupingScalarFunction) groupingScalarFunction.accept(this, rewriteContext); + if (!rewriteContext.isValid()) { + return slot; + } + return Repeat.generateVirtualSlotByFunction(groupingScalarFunction); + } + } if (rewriteContext.getMvExprToMvScanExprQueryBasedMapping().containsKey(slot)) { return rewriteContext.getMvExprToMvScanExprQueryBasedMapping().get(slot); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index d7d6a634f7a4ce..88481c44d730b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; @@ -314,7 +315,18 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } } } + List rewrittenPlanOutput = rewrittenPlan.getOutput(); rewrittenPlan = normalizeExpressions(rewrittenPlan, queryPlan); + if (rewrittenPlan == null) { + // maybe virtual slot reference added automatically + materializationContext.recordFailReason(queryStructInfo, + "RewrittenPlan output logical properties is different with target group", + () -> String.format("materialized view rule normalizeExpressions, output size between " + + "origin and rewritten plan is different, rewritten output is %s, " + + "origin output is %s", + rewrittenPlanOutput, queryPlan.getOutput())); + continue; + } if (!isOutputValid(queryPlan, rewrittenPlan)) { LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties(); materializationContext.recordFailReason(queryStructInfo, @@ -351,6 +363,9 @@ private boolean needUnionRewrite( // Normalize expression such as nullable property and output slot id protected Plan normalizeExpressions(Plan rewrittenPlan, Plan originPlan) { + if (rewrittenPlan.getOutput().size() != originPlan.getOutput().size()) { + return null; + } // normalize nullable List normalizeProjects = new ArrayList<>(); for (int i = 0; i < originPlan.getOutput().size(); i++) { @@ -701,6 +716,11 @@ protected boolean checkIfRewritten(Plan plan, MaterializationContext context) { && context.alreadyRewrite(plan.getGroupExpression().get().getOwnerGroup().getGroupId()); } + protected boolean isEmptyVirtualSlot(Expression expression) { + return expression instanceof VirtualSlotReference + && ((VirtualSlotReference) expression).getRealExpressions().isEmpty(); + } + // check mv plan is valid or not, this can use cache for performance private boolean isMaterializationValid(CascadesContext cascadesContext, MaterializationContext context) { long materializationId = context.getMaterializationQualifier().hashCode(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 077262f0356e66..5d9f6f85689c83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -20,7 +20,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.CascadesContext; @@ -62,12 +61,18 @@ public void initMaterializationContext(CascadesContext cascadesContext) { if (!cascadesContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite()) { return; } - Plan rewritePlan = cascadesContext.getRewritePlan(); TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), true); - // Keep use one connection context when in query, if new connect context, - // the ConnectionContext.get() will change - collectorContext.setConnectContext(cascadesContext.getConnectContext()); - rewritePlan.accept(TableCollector.INSTANCE, collectorContext); + try { + Plan rewritePlan = cascadesContext.getRewritePlan(); + // Keep use one connection context when in query, if new connect context, + // the ConnectionContext.get() will change + collectorContext.setConnectContext(cascadesContext.getConnectContext()); + rewritePlan.accept(TableCollector.INSTANCE, collectorContext); + } catch (Exception e) { + LOG.warn(String.format("MaterializationContext init table collect fail, current queryId is %s", + cascadesContext.getConnectContext().getQueryIdentifier()), e); + return; + } Set collectedTables = collectorContext.getCollectedTables(); if (collectedTables.isEmpty()) { return; @@ -77,30 +82,32 @@ public void initMaterializationContext(CascadesContext cascadesContext) { Set availableMTMVs = Env.getCurrentEnv().getMtmvService().getRelationManager() .getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext()); if (availableMTMVs.isEmpty()) { - LOG.warn(String.format("enable materialized view rewrite but availableMTMVs is empty, current queryId " - + "is %s", cascadesContext.getConnectContext().getQueryIdentifier())); + LOG.debug(String.format("Enable materialized view rewrite but availableMTMVs is empty, current queryId " + + "is %s", cascadesContext.getConnectContext().getQueryIdentifier())); return; } for (MTMV materializedView : availableMTMVs) { MTMVCache mtmvCache = null; try { mtmvCache = materializedView.getOrGenerateCache(cascadesContext.getConnectContext()); - } catch (AnalysisException e) { - LOG.warn("MaterializationContext init mv cache generate fail", e); - } - if (mtmvCache == null) { - continue; + if (mtmvCache == null) { + continue; + } + // For async materialization context, the cascades context when construct the struct info maybe + // different from the current cascadesContext + // so regenerate the struct info table bitset + StructInfo mvStructInfo = mtmvCache.getStructInfo(); + BitSet tableBitSetInCurrentCascadesContext = new BitSet(); + mvStructInfo.getRelations().forEach(relation -> tableBitSetInCurrentCascadesContext.set( + cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt())); + cascadesContext.addMaterializationContext(new AsyncMaterializationContext(materializedView, + mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), ImmutableList.of(), + ImmutableList.of(), cascadesContext, + mtmvCache.getStructInfo().withTableBitSet(tableBitSetInCurrentCascadesContext))); + } catch (Exception e) { + LOG.warn(String.format("MaterializationContext init mv cache generate fail, current queryId is %s", + cascadesContext.getConnectContext().getQueryIdentifier()), e); } - // For async materialization context, the cascades context when construct the struct info maybe - // different from the current cascadesContext - // so regenerate the struct info table bitset - StructInfo mvStructInfo = mtmvCache.getStructInfo(); - BitSet tableBitSetInCurrentCascadesContext = new BitSet(); - mvStructInfo.getRelations().forEach(relation -> tableBitSetInCurrentCascadesContext.set( - cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt())); - cascadesContext.addMaterializationContext(new AsyncMaterializationContext(materializedView, - mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), ImmutableList.of(), ImmutableList.of(), - cascadesContext, mtmvCache.getStructInfo().withTableBitSet(tableBitSetInCurrentCascadesContext))); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 2e45fa44833276..7da1f91c950583 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -195,7 +195,7 @@ public static List extractStructInfo(Plan plan, CascadesContext casc * should be different */ public static Plan generateMvScanPlan(MTMV materializedView, CascadesContext cascadesContext) { - LogicalOlapScan mvScan = new LogicalOlapScan( + return new LogicalOlapScan( cascadesContext.getStatementContext().getNextRelationId(), materializedView, materializedView.getFullQualifiers(), @@ -205,9 +205,6 @@ public static Plan generateMvScanPlan(MTMV materializedView, CascadesContext cas // this must be empty, or it will be used to sample ImmutableList.of(), Optional.empty()); - List mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) - .collect(Collectors.toList()); - return new LogicalProject(mvProjects, mvScan); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index d23806339c628c..4dcb74571192fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -54,6 +54,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; @@ -607,9 +608,6 @@ public Boolean visitLogicalAggregate(LogicalAggregate aggregate, checkContext.setContainsTopAggregate(true); checkContext.plusTopAggregateNum(); } - if (aggregate.getSourceRepeat().isPresent()) { - return false; - } return visit(aggregate, checkContext); } @@ -627,7 +625,8 @@ public Boolean visit(Plan plan, PlanCheckContext checkContext) { || plan instanceof Join || plan instanceof LogicalSort || plan instanceof LogicalAggregate - || plan instanceof GroupPlan) { + || plan instanceof GroupPlan + || plan instanceof LogicalRepeat) { return doVisit(plan, checkContext); } return false; @@ -660,7 +659,8 @@ public Boolean visit(Plan plan, PlanCheckContext checkContext) { if (plan instanceof Filter || plan instanceof Project || plan instanceof CatalogRelation - || plan instanceof GroupPlan) { + || plan instanceof GroupPlan + || plan instanceof LogicalRepeat) { return doVisit(plan, checkContext); } return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index c3d9ce287c2af5..d4d40132417be7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1706,7 +1706,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward = true, description = {"是否开启基于结构信息的物化视图透明改写", "Whether to enable materialized view rewriting based on struct info"}) - public boolean enableMaterializedViewRewrite = false; + public boolean enableMaterializedViewRewrite = true; @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_EXTERNAL_TABLE, needForward = true, description = {"基于结构信息的透明改写，是否使用包含外表的物化视图", diff --git a/regression-test/data/nereids_rules_p0/mv/grouping_sets/grouping_sets.out b/regression-test/data/nereids_rules_p0/mv/grouping_sets/grouping_sets.out new file mode 100644 index 00000000000000..b6e7a2759840e4 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/grouping_sets/grouping_sets.out @@ -0,0 +1,225 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +\N \N \N 178.10 56.20 1.20 8 0 +\N \N a 77.50 33.50 9.50 5 0 +\N \N c 100.60 56.20 1.20 3 0 +o \N \N 178.10 56.20 1.20 8 0 +o 2023-12-08 \N 20.00 10.50 9.50 2 0 +o 2023-12-09 \N 11.50 11.50 11.50 1 0 +o 2023-12-10 \N 46.00 33.50 12.50 2 0 +o 2023-12-11 \N 43.20 43.20 43.20 1 0 +o 2023-12-12 \N 57.40 56.20 1.20 2 0 + +-- !query1_0_after -- +\N \N \N 178.10 56.20 1.20 8 0 +\N \N a 77.50 33.50 9.50 5 0 +\N \N c 100.60 56.20 1.20 3 0 +o \N \N 178.10 56.20 1.20 8 0 +o 2023-12-08 \N 20.00 10.50 9.50 2 0 +o 2023-12-09 \N 11.50 11.50 11.50 1 0 +o 2023-12-10 \N 46.00 33.50 12.50 2 0 +o 2023-12-11 \N 43.20 43.20 43.20 1 0 +o 2023-12-12 \N 57.40 56.20 1.20 2 0 + +-- !query2_0_before -- +\N \N \N 7 3 1 178.10 56.20 1.20 8 0 +\N \N a 6 3 1 77.50 33.50 9.50 5 0 +\N \N c 6 3 1 100.60 56.20 1.20 3 0 +o \N \N 3 1 1 178.10 56.20 1.20 8 0 +o 2023-12-08 \N 1 0 0 20.00 10.50 9.50 2 0 +o 2023-12-09 \N 1 0 0 11.50 11.50 11.50 1 0 +o 2023-12-10 \N 1 0 0 46.00 33.50 12.50 2 0 +o 2023-12-11 \N 1 0 0 43.20 43.20 43.20 1 0 +o 2023-12-12 \N 1 0 0 57.40 56.20 1.20 2 0 + +-- !query2_0_after -- +\N \N \N 7 3 1 178.10 56.20 1.20 8 0 +\N \N a 6 3 1 77.50 33.50 9.50 5 0 +\N \N c 6 3 1 100.60 56.20 1.20 3 0 +o \N \N 3 1 1 178.10 56.20 1.20 8 0 +o 2023-12-08 \N 1 0 0 20.00 10.50 9.50 2 0 +o 2023-12-09 \N 1 0 0 11.50 11.50 11.50 1 0 +o 2023-12-10 \N 1 0 0 46.00 33.50 12.50 2 0 +o 2023-12-11 \N 1 0 0 43.20 43.20 43.20 1 0 +o 2023-12-12 \N 1 0 0 57.40 56.20 1.20 2 0 + +-- !query3_0_before -- +\N \N \N 43.20 43.20 43.20 1 0 +\N 3 \N 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 43.20 43.20 43.20 1 0 +3 3 \N 43.20 43.20 43.20 1 0 + +-- !query3_0_after -- +\N \N \N 43.20 43.20 43.20 1 0 +\N 3 \N 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 43.20 43.20 43.20 1 0 +3 3 \N 43.20 43.20 43.20 1 0 + +-- !query4_0_before -- +\N \N \N 1 1 3 7 43.20 43.20 43.20 1 0 +\N 3 \N 0 1 2 5 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 1 0 1 2 43.20 43.20 43.20 1 0 +3 3 \N 0 1 0 1 43.20 43.20 43.20 1 0 + +-- !query4_0_after -- +\N \N \N 1 1 3 7 43.20 43.20 43.20 1 0 +\N 3 \N 0 1 2 5 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 1 0 1 2 43.20 43.20 43.20 1 0 +3 3 \N 0 1 0 1 43.20 43.20 43.20 1 0 + +-- !query5_0_before -- +\N \N 178.10 56.20 1.20 8 0 +\N a 77.50 33.50 9.50 5 0 +\N c 100.60 56.20 1.20 3 0 +o \N 178.10 56.20 1.20 8 0 +o a 77.50 33.50 9.50 5 0 +o c 100.60 56.20 1.20 3 0 + +-- !query5_0_after -- +\N \N 178.10 56.20 1.20 8 0 +\N a 77.50 33.50 9.50 5 0 +\N c 100.60 56.20 1.20 3 0 +o \N 178.10 56.20 1.20 8 0 +o a 77.50 33.50 9.50 5 0 +o c 100.60 56.20 1.20 3 0 + +-- !query6_0_before -- +\N \N 3 1 1 178.10 56.20 1.20 8 0 +\N a 2 1 1 77.50 33.50 9.50 5 0 +\N c 2 1 1 100.60 56.20 1.20 3 0 +o \N 1 0 0 178.10 56.20 1.20 8 0 +o a 0 0 0 77.50 33.50 9.50 5 0 +o c 0 0 0 100.60 56.20 1.20 3 0 + +-- !query6_0_after -- +\N \N 3 1 1 178.10 56.20 1.20 8 0 +\N a 2 1 1 77.50 33.50 9.50 5 0 +\N c 2 1 1 100.60 56.20 1.20 3 0 +o \N 1 0 0 178.10 56.20 1.20 8 0 +o a 0 0 0 77.50 33.50 9.50 5 0 +o c 0 0 0 100.60 56.20 1.20 3 0 + +-- !query7_0_before -- +\N \N \N 43.20 43.20 43.20 1 0 +\N \N 2023-12-11 43.20 43.20 43.20 1 0 +\N 3 \N 43.20 43.20 43.20 1 0 +\N 3 2023-12-11 43.20 43.20 43.20 1 0 +3 \N \N 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 43.20 43.20 43.20 1 0 +3 3 \N 43.20 43.20 43.20 1 0 +3 3 2023-12-11 43.20 43.20 43.20 1 0 + +-- !query7_0_after -- +\N \N \N 43.20 43.20 43.20 1 0 +\N \N 2023-12-11 43.20 43.20 43.20 1 0 +\N 3 \N 43.20 43.20 43.20 1 0 +\N 3 2023-12-11 43.20 43.20 43.20 1 0 +3 \N \N 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 43.20 43.20 43.20 1 0 +3 3 \N 43.20 43.20 43.20 1 0 +3 3 2023-12-11 43.20 43.20 43.20 1 0 + +-- !query8_0_before -- +\N \N \N 1 1 3 7 43.20 43.20 43.20 1 0 +\N \N 2023-12-11 1 0 3 6 43.20 43.20 43.20 1 0 +\N 3 \N 0 1 2 5 43.20 43.20 43.20 1 0 +\N 3 2023-12-11 0 0 2 4 43.20 43.20 43.20 1 0 +3 \N \N 1 1 1 3 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 1 0 1 2 43.20 43.20 43.20 1 0 +3 3 \N 0 1 0 1 43.20 43.20 43.20 1 0 +3 3 2023-12-11 0 0 0 0 43.20 43.20 43.20 1 0 + +-- !query8_0_after -- +\N \N \N 1 1 3 7 43.20 43.20 43.20 1 0 +\N \N 2023-12-11 1 0 3 6 43.20 43.20 43.20 1 0 +\N 3 \N 0 1 2 5 43.20 43.20 43.20 1 0 +\N 3 2023-12-11 0 0 2 4 43.20 43.20 43.20 1 0 +3 \N \N 1 1 1 3 43.20 43.20 43.20 1 0 +3 \N 2023-12-11 1 0 1 2 43.20 43.20 43.20 1 0 +3 3 \N 0 1 0 1 43.20 43.20 43.20 1 0 +3 3 2023-12-11 0 0 0 0 43.20 43.20 43.20 1 0 + +-- !query9_0_before -- +\N \N 178.10 56.20 1.20 8 0 +o \N 178.10 56.20 1.20 8 0 +o a 77.50 33.50 9.50 5 0 +o c 100.60 56.20 1.20 3 0 + +-- !query9_0_after -- +\N \N 178.10 56.20 1.20 8 0 +o \N 178.10 56.20 1.20 8 0 +o a 77.50 33.50 9.50 5 0 +o c 100.60 56.20 1.20 3 0 + +-- !query10_0_before -- +\N \N 3 1 1 178.10 56.20 1.20 8 0 +o \N 1 0 0 178.10 56.20 1.20 8 0 +o a 0 0 0 77.50 33.50 9.50 5 0 +o c 0 0 0 100.60 56.20 1.20 3 0 + +-- !query10_0_after -- +\N \N 3 1 1 178.10 56.20 1.20 8 0 +o \N 1 0 0 178.10 56.20 1.20 8 0 +o a 0 0 0 77.50 33.50 9.50 5 0 +o c 0 0 0 100.60 56.20 1.20 3 0 + +-- !query11_0_before -- +\N \N \N 43.20 43.20 43.20 1 0 +3 \N \N 43.20 43.20 43.20 1 0 +3 3 \N 43.20 43.20 43.20 1 0 +3 3 2023-12-11 43.20 43.20 43.20 1 0 + +-- !query11_0_after -- +\N \N \N 43.20 43.20 43.20 1 0 +3 \N \N 43.20 43.20 43.20 1 0 +3 3 \N 43.20 43.20 43.20 1 0 +3 3 2023-12-11 43.20 43.20 43.20 1 0 + +-- !query12_0_before -- +\N \N \N 1 1 3 7 43.20 43.20 43.20 1 0 +3 \N \N 1 1 1 3 43.20 43.20 43.20 1 0 +3 3 \N 0 1 0 1 43.20 43.20 43.20 1 0 +3 3 2023-12-11 0 0 0 0 43.20 43.20 43.20 1 0 + +-- !query12_0_after -- +\N \N \N 1 1 3 7 43.20 43.20 43.20 1 0 +3 \N \N 1 1 1 3 43.20 43.20 43.20 1 0 +3 3 \N 0 1 0 1 43.20 43.20 43.20 1 0 +3 3 2023-12-11 0 0 0 0 43.20 43.20 43.20 1 0 + +-- !query13_0_before -- +\N \N \N 178.10 56.20 1.20 8 0 +\N \N a 77.50 33.50 9.50 5 0 +\N \N c 100.60 56.20 1.20 3 0 +o \N \N 178.10 56.20 1.20 8 0 +o 2023-12-08 \N 20.00 10.50 9.50 2 0 +o 2023-12-09 \N 11.50 11.50 11.50 1 0 +o 2023-12-10 \N 46.00 33.50 12.50 2 0 +o 2023-12-11 \N 43.20 43.20 43.20 1 0 +o 2023-12-12 \N 57.40 56.20 1.20 2 0 + +-- !query13_0_after -- +\N \N \N 178.10 56.20 1.20 8 0 +\N \N a 77.50 33.50 9.50 5 0 +\N \N c 100.60 56.20 1.20 3 0 +o \N \N 178.10 56.20 1.20 8 0 +o 2023-12-08 \N 20.00 10.50 9.50 2 0 +o 2023-12-09 \N 11.50 11.50 11.50 1 0 +o 2023-12-10 \N 46.00 33.50 12.50 2 0 +o 2023-12-11 \N 43.20 43.20 43.20 1 0 +o 2023-12-12 \N 57.40 56.20 1.20 2 0 + +-- !query14_0_before -- +o 2023-12-08 a 20.00 10.50 9.50 2 \N +o 2023-12-09 a 11.50 11.50 11.50 1 \N +o 2023-12-10 a 46.00 33.50 12.50 2 \N +o 2023-12-11 c 43.20 43.20 43.20 1 \N +o 2023-12-12 c 57.40 56.20 1.20 2 \N + +-- !query14_0_after -- +o 2023-12-08 a 20.00 10.50 9.50 2 \N +o 2023-12-09 a 11.50 11.50 11.50 1 \N +o 2023-12-10 a 46.00 33.50 12.50 2 \N +o 2023-12-11 c 43.20 43.20 43.20 1 \N +o 2023-12-12 c 57.40 56.20 1.20 2 \N + diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy index ac1e806bd51224..87c582f80c8d84 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy @@ -18,11 +18,8 @@ suite("aggregate_with_roll_up") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy index b0e82c73d5f449..72a6144bcdaf92 100644 --- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy @@ -18,12 +18,8 @@ suite("aggregate_without_roll_up") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" - sql "SET enable_nereids_timeout = false" sql "SET enable_agg_state = true" sql """ diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy index c40997f89897a1..f4cfbdc4d9a6d4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy @@ -23,10 +23,7 @@ suite("grace_period") { // if update will not be used to query rewrite String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_partition diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy index b4ae8b7bfc544f..4a9784ed04b985 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/materialized_view_switch.groovy @@ -19,11 +19,8 @@ suite("materialized_view_switch") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy index e7f4f92cd06786..9b5aa76b5baa2b 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_1.groovy @@ -21,9 +21,6 @@ This suite is a one dimensional test case file. suite("partition_mv_rewrite_dimension_1") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_1 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy index d261e7c4160a81..91b50ecd0d2078 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_3.groovy @@ -22,9 +22,6 @@ It mainly tests the agg function, etc suite("partition_mv_rewrite_dimension_2_3") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_3 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy index 5215ab75d20be4..9535fc2e4ad4a5 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_4.groovy @@ -22,9 +22,6 @@ It mainly tests the query partial, view partial, union rewriting, predicate comp suite("partition_mv_rewrite_dimension_2_4") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_4 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy index e392a5178011c6..6f5d4ce42ccb51 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_5.groovy @@ -22,9 +22,6 @@ It mainly tests the query partial, view partial, union rewriting, predicate comp suite("partition_mv_rewrite_dimension_2_5") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_5 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy index 12c7a23448b374..39c3d13e09b900 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_6.groovy @@ -22,9 +22,6 @@ It mainly tests the query partial, view partial, union rewriting, predicate comp suite("partition_mv_rewrite_dimension_2_6") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_6 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy index a4572bb5e99f96..e38ef749ea8ed6 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_full_join.groovy @@ -22,9 +22,6 @@ It mainly tests the full join and filter positions. suite("partition_mv_rewrite_dimension_2_full_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_full_join diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy index 97dcdc279e3264..932f36f9c78bbd 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_inner_join.groovy @@ -22,9 +22,6 @@ It mainly tests the inner join and filter positions. suite("partition_mv_rewrite_dimension_2_2") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_2 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy index f62d24db6a4227..14daf54727d103 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_anti_join.groovy @@ -22,9 +22,6 @@ It mainly tests the left anti join and filter positions. suite("partition_mv_rewrite_dimension_2_left_anti_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_left_anti_join diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy index d5be7599d222ca..043625335dfaa4 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_join.groovy @@ -22,9 +22,6 @@ It mainly tests the left join and filter positions. suite("partition_mv_rewrite_dimension_2_1") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_1 diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy index bb7841297e02ec..e5bb974a5b133a 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_left_semi_join.groovy @@ -22,9 +22,6 @@ It mainly tests the left semi join and filter positions. suite("partition_mv_rewrite_dimension_2_left_semi_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_left_semi_join diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy index 2b60e58927c780..957a1b45a080e5 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_anti_join.groovy @@ -22,9 +22,6 @@ It mainly tests the right anti join and filter positions. suite("partition_mv_rewrite_dimension_2_right_anti_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_right_anti_join diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy index ab12b4eb8c7a7d..83c41fd0fd4ff2 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_join.groovy @@ -22,9 +22,6 @@ It mainly tests the right join and filter positions. suite("partition_mv_rewrite_dimension_2_right_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_right_join diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy index 0d2a199356c8ed..c1307667685fdc 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_2_right_semi_join.groovy @@ -22,9 +22,6 @@ It mainly tests the right semi join and filter positions. suite("partition_mv_rewrite_dimension_2_right_semi_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_2_right_semi_join diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy index 4812c98a4a3532..8db90bc40ebaa3 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy @@ -21,9 +21,6 @@ This suite test self connection case suite("partition_mv_rewrite_dimension_self_conn") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_self_conn diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy index f1916220984e7b..4d28ea5f3edd1e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_2_join_agg/dimension_2_join_agg.groovy @@ -21,9 +21,6 @@ This file is used specifically to test the presence of AGGs under joins. suite("dimension_2_join_agg_replenish") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy index bddf6c22d3ba4b..fed1ec06ed7e42 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dimension_equal/filter_equal_or_notequal.groovy @@ -19,9 +19,6 @@ suite("filter_equal_or_notequal_case") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_1 diff --git a/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy new file mode 100644 index 00000000000000..6882a279b2ded2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/grouping_sets/grouping_sets.groovy @@ -0,0 +1,612 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("materialized_view_grouping_sets") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + O_COMMENT VARCHAR(79) NOT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + + sql """ + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) ( + PARTITION `day_1` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_2` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_3` VALUES LESS THAN ("2023-12-30")) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + drop table if exists partsupp + """ + + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy'), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx'); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1'), + (2, 3, 10, 11.01, 'supply2'); + """ + + // query has group sets, and mv doesn't + // single table grouping sets without grouping scalar function + def mv1_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from orders + group by + o_orderstatus, o_orderdate, o_orderpriority; + """ + def query1_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + GROUPING SETS ((o_orderstatus, o_orderdate), (o_orderpriority), (o_orderstatus), ()); + """ + order_qt_query1_0_before "${query1_0}" + check_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" + + // single table grouping sets with grouping scalar function + def mv2_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from orders + group by + o_orderstatus, o_orderdate, o_orderpriority; + """ + def query2_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + grouping_id(o_orderstatus, o_orderdate, o_orderpriority), + grouping_id(o_orderstatus, o_orderdate), + grouping(o_orderdate), + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + GROUPING SETS ((o_orderstatus, o_orderdate), (o_orderpriority), (o_orderstatus), ()); + """ + order_qt_query2_0_before "${query2_0}" + check_mv_rewrite_success(db, mv2_0, query2_0, "mv2_0") + order_qt_query2_0_after "${query2_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv2_0""" + + + // multi table grouping sets without grouping scalar function + def mv3_0 = + """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query3_0 = + """ + select t1.l_partkey, t1.l_suppkey, o_orderdate, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + group by + GROUPING SETS ((l_shipdate, o_orderdate, l_partkey), (l_partkey, l_suppkey), (l_suppkey), ()); + """ + order_qt_query3_0_before "${query3_0}" + check_mv_rewrite_success(db, mv3_0, query3_0, "mv3_0") + order_qt_query3_0_after "${query3_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv3_0""" + + + // multi table grouping sets with grouping scalar function + def mv4_0 = + """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query4_0 = + """ + select t1.l_partkey, t1.l_suppkey, o_orderdate, + grouping(t1.l_suppkey), + grouping(o_orderdate), + grouping_id(t1.l_partkey, t1.l_suppkey), + grouping_id(t1.l_partkey, t1.l_suppkey, o_orderdate), + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + group by + GROUPING SETS ((l_shipdate, o_orderdate, l_partkey), (l_partkey, l_suppkey), (l_suppkey), ()); + """ + order_qt_query4_0_before "${query4_0}" + check_mv_rewrite_success(db, mv4_0, query4_0, "mv4_0") + order_qt_query4_0_after "${query4_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv4_0""" + + + // single table cube without grouping scalar function + def mv5_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from orders + group by + o_orderstatus, o_orderdate, o_orderpriority; + """ + def query5_0 = + """ + select o_orderstatus, o_orderpriority, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + CUBE (o_orderstatus, o_orderpriority); + """ + order_qt_query5_0_before "${query5_0}" + check_mv_rewrite_success(db, mv5_0, query5_0, "mv5_0") + order_qt_query5_0_after "${query5_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv5_0""" + + // single table cube with grouping scalar function + def mv6_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from orders + group by + o_orderstatus, o_orderdate, o_orderpriority; + """ + def query6_0 = + """ + select o_orderstatus, o_orderpriority, + grouping_id(o_orderstatus, o_orderpriority), + grouping_id(o_orderstatus), + grouping(o_orderstatus), + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + CUBE (o_orderstatus, o_orderpriority); + """ + order_qt_query6_0_before "${query6_0}" + check_mv_rewrite_success(db, mv6_0, query6_0, "mv6_0") + order_qt_query6_0_after "${query6_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_0""" + + // multi table cube without grouping scalar function + def mv7_0 = + """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query7_0 = + """ + select t1.l_partkey, t1.l_suppkey, o_orderdate, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + group by + CUBE (t1.l_partkey, t1.l_suppkey, o_orderdate); + """ + order_qt_query7_0_before "${query7_0}" + check_mv_rewrite_success(db, mv7_0, query7_0, "mv7_0") + order_qt_query7_0_after "${query7_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_0""" + + // multi table cube with grouping scalar function + def mv8_0 = + """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query8_0 = + """ + select t1.l_partkey, t1.l_suppkey, o_orderdate, + grouping(t1.l_suppkey), + grouping(o_orderdate), + grouping_id(t1.l_partkey, t1.l_suppkey), + grouping_id(t1.l_partkey, t1.l_suppkey, o_orderdate), + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + group by + CUBE (t1.l_partkey, t1.l_suppkey, o_orderdate); + """ + order_qt_query8_0_before "${query8_0}" + check_mv_rewrite_success(db, mv8_0, query8_0, "mv8_0") + order_qt_query8_0_after "${query8_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv8_0""" + + + + // single table rollup without grouping scalar function + def mv9_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from orders + group by + o_orderstatus, o_orderdate, o_orderpriority; + """ + def query9_0 = + """ + select o_orderstatus, o_orderpriority, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + ROLLUP (o_orderstatus, o_orderpriority); + """ + order_qt_query9_0_before "${query9_0}" + check_mv_rewrite_success(db, mv9_0, query9_0, "mv9_0") + order_qt_query9_0_after "${query9_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv9_0""" + + // single table rollup with grouping scalar function + def mv10_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from orders + group by + o_orderstatus, o_orderdate, o_orderpriority; + """ + def query10_0 = + """ + select o_orderstatus, o_orderpriority, + grouping_id(o_orderstatus, o_orderpriority), + grouping_id(o_orderstatus), + grouping(o_orderstatus), + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + ROLLUP (o_orderstatus, o_orderpriority); + """ + order_qt_query10_0_before "${query10_0}" + check_mv_rewrite_success(db, mv10_0, query10_0, "mv10_0") + order_qt_query10_0_after "${query10_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv10_0""" + + // multi table rollup without grouping scalar function + def mv11_0 = + """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query11_0 = + """ + select t1.l_partkey, t1.l_suppkey, o_orderdate, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + group by + ROLLUP (t1.l_partkey, t1.l_suppkey, o_orderdate); + """ + order_qt_query11_0_before "${query11_0}" + check_mv_rewrite_success(db, mv11_0, query11_0, "mv11_0") + order_qt_query11_0_after "${query11_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv11_0""" + + // multi table rollup with grouping scalar function + def mv12_0 = + """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query12_0 = + """ + select t1.l_partkey, t1.l_suppkey, o_orderdate, + grouping(t1.l_suppkey), + grouping(o_orderdate), + grouping_id(t1.l_partkey, t1.l_suppkey), + grouping_id(t1.l_partkey, t1.l_suppkey, o_orderdate), + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from (select * from lineitem where l_shipdate = '2023-12-11') t1 + left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate + group by + ROLLUP (t1.l_partkey, t1.l_suppkey, o_orderdate); + """ + order_qt_query12_0_before "${query12_0}" + check_mv_rewrite_success(db, mv12_0, query12_0, "mv12_0") + order_qt_query12_0_after "${query12_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv12_0""" + + + // both query and mv has group sets + // not support + def mv13_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + GROUPING SETS ((o_orderstatus, o_orderdate), (o_orderpriority), (o_orderstatus), ()); + """ + def query13_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + GROUPING SETS ((o_orderstatus, o_orderdate), (o_orderpriority), (o_orderstatus), ()); + """ + order_qt_query13_0_before "${query13_0}" + check_mv_rewrite_fail(db, mv13_0, query13_0, "mv13_0") + order_qt_query13_0_after "${query13_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv13_0""" + + + // mv has group sets, and query doesn't + // not support + def mv14_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice), + max(o_totalprice), + min(o_totalprice), + count(*), + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) + from orders + group by + GROUPING SETS ((o_orderstatus, o_orderdate), (o_orderpriority), (o_orderstatus), ()); + """ + def query14_0 = + """ + select o_orderstatus, o_orderdate, o_orderpriority, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic + from orders + group by + o_orderstatus, o_orderdate, o_orderpriority; + """ + order_qt_query14_0_before "${query14_0}" + check_mv_rewrite_fail(db, mv14_0, query14_0, "mv14_0") + order_qt_query14_0_after "${query14_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv14_0""" +} + diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy index 2c3eb9fe3dd641..75b5276e442747 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_inner/inner_join_dphyp.groovy @@ -18,10 +18,7 @@ suite("inner_join_dphyp") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql "SET enable_dphyp_optimizer = true" sql """ diff --git a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy index d56888e53a0637..048d802e274b68 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/dphyp_outer/outer_join_dphyp.groovy @@ -18,11 +18,8 @@ suite("outer_join_dphyp") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql "SET enable_dphyp_optimizer = true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy index 29392c4f5aa287..6a990b67b72cd0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy @@ -18,10 +18,7 @@ suite("inner_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy index 0b48642097d8cc..68e323f1eb9af0 100644 --- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy @@ -18,11 +18,8 @@ suite("outer_join") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF"; sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy index 4b60ee60ccd3ed..c8ef5729d4b704 100644 --- a/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/negative/negative_test.groovy @@ -21,9 +21,6 @@ This suite is a one dimensional test case file. suite("negative_partition_mv_rewrite") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_1 diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy index 235c6a246cf125..6219cc7313d79e 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy @@ -98,10 +98,7 @@ suite("nested_materialized_view") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite = true" def create_mtmv = { db_name, mv_name, mv_sql -> diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy index ec3d2912df96b4..2d81ef893524e1 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy @@ -20,10 +20,7 @@ import java.text.SimpleDateFormat suite("partition_mv_rewrite") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy index 27a0292dfe01b0..27c625194a84bf 100644 --- a/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/scan/scan_table.groovy @@ -18,10 +18,7 @@ suite("mv_scan_table") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy b/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy index 5beb23971b6484..674789f4304ffb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy @@ -86,10 +86,7 @@ suite("mv_ssb_test") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" sql "SET BATCH_SIZE = 4064" diff --git a/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy b/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy index 15b9ebf9f23f3a..e033fbebd899fb 100644 --- a/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/tpch/mv_tpch_test.groovy @@ -84,10 +84,7 @@ suite("mv_tpch_test") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" // mv8 Nereids cost too much time ( > 5s ) sql "SET enable_nereids_timeout=false" diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy index 90d326ed7ec899..2c67a373664089 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/partition_curd_union_rewrite.groovy @@ -18,10 +18,7 @@ suite ("partition_curd_union_rewrite") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy index 30597d57e1c415..e31b4d41a3d4af 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy @@ -18,9 +18,6 @@ suite ("usercase_union_rewrite") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" - sql "SET enable_nereids_planner=true" - sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql """ drop table if exists orders_user