Skip to content

[Pick 2.0](inverted index) push count on index down to scan node (#22687) #23794

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiaokang merged 2 commits into from
Sep 5, 2023
Merged

[Pick 2.0](inverted index) push count on index down to scan node (#22687) #23794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
eedbb95
[Feature](inverted index) push count on index down to scan node (#22687)
airborne12 Sep 2, 2023
bd287d6
[regression](inverted index) force nereids open for count on index
airborne12 Sep 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@ public enum RuleType {
STORAGE_LAYER_AGGREGATE_WITH_PROJECT(RuleTypeClass.IMPLEMENTATION),
STORAGE_LAYER_AGGREGATE_WITH_PROJECT_FOR_FILE_SCAN(RuleTypeClass.IMPLEMENTATION),
COUNT_ON_INDEX(RuleTypeClass.IMPLEMENTATION),
COUNT_ON_INDEX_WITHOUT_PROJECT(RuleTypeClass.IMPLEMENTATION),
ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT(RuleTypeClass.IMPLEMENTATION),
TWO_PHASE_AGGREGATE_WITHOUT_DISTINCT(RuleTypeClass.IMPLEMENTATION),
TWO_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI(RuleTypeClass.IMPLEMENTATION),
Expand Down
60 changes: 47 additions & 13 deletions ...core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
Expand All @@ -99,14 +100,31 @@ public List<Rule> buildRules() {
PatternDescriptor<LogicalAggregate<GroupPlan>> basePattern = logicalAggregate();

return ImmutableList.of(
RuleType.COUNT_ON_INDEX_WITHOUT_PROJECT.build(
logicalAggregate(
logicalFilter(
logicalOlapScan().when(this::isDupOrMowKeyTable)
).when(filter -> containsMatchExpression(filter.getConjuncts())))
.when(agg -> enablePushDownCountOnIndex())
.when(agg -> agg.getGroupByExpressions().size() == 0)
.when(agg -> {
Set<AggregateFunction> funcs = agg.getAggregateFunctions();
return !funcs.isEmpty() && funcs.stream()
.allMatch(f -> f instanceof Count && !f.isDistinct());
})
.thenApply(ctx -> {
LogicalAggregate<LogicalFilter<LogicalOlapScan>> agg = ctx.root;
LogicalFilter<LogicalOlapScan> filter = agg.child();
LogicalOlapScan olapScan = filter.child();
return pushdownCountOnIndex(agg, null, filter, olapScan, ctx.cascadesContext);
})
),
RuleType.COUNT_ON_INDEX.build(
logicalAggregate(
logicalProject(
logicalFilter(
logicalOlapScan().when(this::isDupOrMowKeyTable)
).when(filter -> containsMatchExpression(filter.getExpressions())
&& filter.getExpressions().size() == 1)
))
).when(filter -> containsMatchExpression(filter.getConjuncts()))))
.when(agg -> enablePushDownCountOnIndex())
.when(agg -> agg.getGroupByExpressions().size() == 0)
.when(agg -> {
Expand Down Expand Up @@ -212,8 +230,16 @@ && couldConvertToMulti(agg))
);
}

private boolean containsMatchExpression(List<Expression> expressions) {
return expressions.stream().allMatch(expr -> expr instanceof Match);
private boolean containsMatchExpression(Set<Expression> expressions) {
List<Expression> exprs = new ArrayList<>();
expressions.forEach(conjunct -> {
conjunct.getInputSlots().forEach(slot -> {
if (!slot.getName().equalsIgnoreCase(Column.DELETE_SIGN)) {
exprs.add(conjunct);
}
});
});
return exprs.stream().allMatch(expr -> expr instanceof Match);
}

private boolean enablePushDownCountOnIndex() {
Expand Down Expand Up @@ -252,7 +278,7 @@ private boolean isDupOrMowKeyTable(LogicalOlapScan logicalScan) {
*/
private LogicalAggregate<? extends Plan> pushdownCountOnIndex(
LogicalAggregate<? extends Plan> agg,
LogicalProject<? extends Plan> project,
@Nullable LogicalProject<? extends Plan> project,
LogicalFilter<? extends Plan> filter,
LogicalOlapScan olapScan,
CascadesContext cascadesContext) {
Expand All @@ -261,13 +287,21 @@ private LogicalAggregate<? extends Plan> pushdownCountOnIndex(
.build()
.transform(olapScan, cascadesContext)
.get(0);
return agg.withChildren(ImmutableList.of(
project.withChildren(ImmutableList.of(
filter.withChildren(ImmutableList.of(
new PhysicalStorageLayerAggregate(
physicalOlapScan,
PushDownAggOp.COUNT_ON_MATCH)))))
));
if (project != null) {
return agg.withChildren(ImmutableList.of(
project.withChildren(ImmutableList.of(
filter.withChildren(ImmutableList.of(
new PhysicalStorageLayerAggregate(
physicalOlapScan,
PushDownAggOp.COUNT_ON_MATCH)))))
));
} else {
return agg.withChildren(ImmutableList.of(
filter.withChildren(ImmutableList.of(
new PhysicalStorageLayerAggregate(
physicalOlapScan,
PushDownAggOp.COUNT_ON_MATCH)))));
}
}

/**
Expand Down
Loading