Skip to content

[improvement](mtmv) Support grouping_sets rewrite when query rewrite by materialized view #35976

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
seawinde wants to merge 4 commits into from
Closed

[improvement](mtmv) Support grouping_sets rewrite when query rewrite by materialized view #35976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3c14170
[opt](mtmv) Set query rewrite by materialized view default enable
seawinde Jun 5, 2024
2ca4319
[improvement](mtmv) Support agg group sets rewrite
seawinde Jun 3, 2024
e763b8a
[fix](bit shift) fix undefined behavior of bit shift functions. (#35723)
zhiqiang-hhhh Jun 3, 2024
be6fe8a
[improvement](mtmv) Support grouping_sets rewrite when query rewrite …
seawinde Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1577,6 +1577,11 @@ private void transferToMaster() {
SessionVariable.NEREIDS_TIMEOUT_SECOND, "30");
}
}
if (journalVersion <= FeMetaVersion.VERSION_133) {
VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1",
SessionVariable.ENABLE_MATERIALIZED_VIEW_REWRITE,
"true");
}
}

getPolicyMgr().createDefaultStoragePolicy();
Expand Down
45 changes: 27 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,16 +88,23 @@ public Rule build() {
return new LogicalAggregate<>(repeat.getGroupByExpressions(),
repeat.getOutputExpressions(), repeat.child());
}
checkRepeatLegality(repeat);
repeat = removeDuplicateColumns(repeat);
// add virtual slot, LogicalAggregate and LogicalProject for normalize
LogicalAggregate<Plan> agg = normalizeRepeat(repeat);
return dealSlotAppearBothInAggFuncAndGroupingSets(agg);
return doNormalize(repeat);
})
);
}

private LogicalRepeat<Plan> removeDuplicateColumns(LogicalRepeat<Plan> repeat) {
/**
* Normalize repeat, this can be used directly, if optimize the repeat
*/
public static LogicalAggregate<Plan> doNormalize(LogicalRepeat<Plan> repeat) {
checkRepeatLegality(repeat);
repeat = removeDuplicateColumns(repeat);
// add virtual slot, LogicalAggregate and LogicalProject for normalize
LogicalAggregate<Plan> agg = normalizeRepeat(repeat);
return dealSlotAppearBothInAggFuncAndGroupingSets(agg);
}

private static LogicalRepeat<Plan> removeDuplicateColumns(LogicalRepeat<Plan> repeat) {
List<List<Expression>> groupingSets = repeat.getGroupingSets();
ImmutableList.Builder<List<Expression>> builder = ImmutableList.builder();
for (List<Expression> sets : groupingSets) {
Expand All @@ -107,11 +114,11 @@ private LogicalRepeat<Plan> removeDuplicateColumns(LogicalRepeat<Plan> repeat) {
return repeat.withGroupSets(builder.build());
}

private void checkRepeatLegality(LogicalRepeat<Plan> repeat) {
private static void checkRepeatLegality(LogicalRepeat<Plan> repeat) {
checkGroupingSetsSize(repeat);
}

private void checkGroupingSetsSize(LogicalRepeat<Plan> repeat) {
private static void checkGroupingSetsSize(LogicalRepeat<Plan> repeat) {
Set<Expression> flattenGroupingSetExpr = ImmutableSet.copyOf(
ExpressionUtils.flatExpressions(repeat.getGroupingSets()));
if (flattenGroupingSetExpr.size() > LogicalRepeat.MAX_GROUPING_SETS_NUM) {
Expand All @@ -121,7 +128,7 @@ private void checkGroupingSetsSize(LogicalRepeat<Plan> repeat) {
}
}

private LogicalAggregate<Plan> normalizeRepeat(LogicalRepeat<Plan> repeat) {
private static LogicalAggregate<Plan> normalizeRepeat(LogicalRepeat<Plan> repeat) {
Set<Expression> needToSlotsGroupingExpr = collectNeedToSlotGroupingExpr(repeat);
NormalizeToSlotContext groupingExprContext = buildContext(repeat, needToSlotsGroupingExpr);
Map<Expression, NormalizeToSlotTriplet> groupingExprMap = groupingExprContext.getNormalizeToSlotMap();
Expand Down Expand Up @@ -195,14 +202,14 @@ private LogicalAggregate<Plan> normalizeRepeat(LogicalRepeat<Plan> repeat) {
Optional.of(normalizedRepeat), normalizedRepeat);
}

private Set<Expression> collectNeedToSlotGroupingExpr(LogicalRepeat<Plan> repeat) {
private static Set<Expression> collectNeedToSlotGroupingExpr(LogicalRepeat<Plan> repeat) {
// grouping sets should be pushed down, e.g. grouping sets((k + 1)),
// we should push down the `k + 1` to the bottom plan
return ImmutableSet.copyOf(
ExpressionUtils.flatExpressions(repeat.getGroupingSets()));
}

private Set<Expression> collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(LogicalRepeat<Plan> repeat) {
private static Set<Expression> collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(LogicalRepeat<Plan> repeat) {
Set<GroupingScalarFunction> groupingScalarFunctions = ExpressionUtils.collect(
repeat.getOutputExpressions(), GroupingScalarFunction.class::isInstance);
ImmutableSet.Builder<Expression> argumentsSetBuilder = ImmutableSet.builder();
Expand Down Expand Up @@ -234,15 +241,15 @@ private Set<Expression> collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(Logi
.build();
}

private Plan pushDownProject(Set<NamedExpression> pushedExprs, Plan originBottomPlan) {
private static Plan pushDownProject(Set<NamedExpression> pushedExprs, Plan originBottomPlan) {
if (!pushedExprs.equals(originBottomPlan.getOutputSet()) && !pushedExprs.isEmpty()) {
return new LogicalProject<>(ImmutableList.copyOf(pushedExprs), originBottomPlan);
}
return originBottomPlan;
}

/** buildContext */
public NormalizeToSlotContext buildContext(Repeat<? extends Plan> repeat,
public static NormalizeToSlotContext buildContext(Repeat<? extends Plan> repeat,
Set<? extends Expression> sourceExpressions) {
Set<Alias> aliases = ExpressionUtils.collect(repeat.getOutputExpressions(), Alias.class::isInstance);
Map<Expression, Alias> existsAliasMap = Maps.newLinkedHashMap();
Expand All @@ -268,15 +275,16 @@ public NormalizeToSlotContext buildContext(Repeat<? extends Plan> repeat,
return new NormalizeToSlotContext(normalizeToSlotMap);
}

private Optional<NormalizeToSlotTriplet> toGroupingSetExpressionPushDownTriplet(
private static Optional<NormalizeToSlotTriplet> toGroupingSetExpressionPushDownTriplet(
Expression expression, @Nullable Alias existsAlias) {
NormalizeToSlotTriplet originTriplet = NormalizeToSlotTriplet.toTriplet(expression, existsAlias);
SlotReference remainSlot = (SlotReference) originTriplet.remainExpr;
Slot newSlot = remainSlot.withNullable(true);
return Optional.of(new NormalizeToSlotTriplet(expression, newSlot, originTriplet.pushedExpr));
}

private Expression normalizeAggFuncChildrenAndGroupingScalarFunc(NormalizeToSlotContext context, Expression expr) {
private static Expression normalizeAggFuncChildrenAndGroupingScalarFunc(NormalizeToSlotContext context,
Expression expr) {
if (expr instanceof AggregateFunction) {
AggregateFunction function = (AggregateFunction) expr;
List<Expression> normalizedRealExpressions = context.normalizeToUseSlotRef(function.getArguments());
Expand All @@ -293,7 +301,7 @@ private Expression normalizeAggFuncChildrenAndGroupingScalarFunc(NormalizeToSlot
}
}

private Set<Alias> getExistsAlias(LogicalRepeat<Plan> repeat,
private static Set<Alias> getExistsAlias(LogicalRepeat<Plan> repeat,
Map<Expression, NormalizeToSlotTriplet> groupingExprMap) {
Set<Alias> existsAlias = Sets.newHashSet();
Set<Alias> aliases = ExpressionUtils.collect(repeat.getOutputExpressions(), Alias.class::isInstance);
Expand All @@ -320,7 +328,7 @@ private Set<Alias> getExistsAlias(LogicalRepeat<Plan> repeat,
* +--LogicalRepeat (groupingSets=[[a#0]], outputExpr=[a#0, a#3, GROUPING_ID#1]
* +--LogicalProject (projects =[a#0, a#0 as `a`#3])
*/
private LogicalAggregate<Plan> dealSlotAppearBothInAggFuncAndGroupingSets(
private static LogicalAggregate<Plan> dealSlotAppearBothInAggFuncAndGroupingSets(
@NotNull LogicalAggregate<Plan> aggregate) {
LogicalRepeat<Plan> repeat = (LogicalRepeat<Plan>) aggregate.child();
Map<Slot, Alias> commonSlotToAliasMap = getCommonSlotToAliasMap(repeat, aggregate);
Expand Down Expand Up @@ -367,7 +375,8 @@ private LogicalAggregate<Plan> dealSlotAppearBothInAggFuncAndGroupingSets(
return aggregate.withAggOutput(newOutputExpressions);
}

private Map<Slot, Alias> getCommonSlotToAliasMap(LogicalRepeat<Plan> repeat, LogicalAggregate<Plan> aggregate) {
private static Map<Slot, Alias> getCommonSlotToAliasMap(LogicalRepeat<Plan> repeat,
LogicalAggregate<Plan> aggregate) {
List<AggregateFunction> aggregateFunctions =
CollectNonWindowedAggFuncs.collect(aggregate.getOutputExpressions());
ImmutableSet.Builder<Slot> aggUsedSlotBuilder = ImmutableSet.builder();
Expand Down
Loading