Skip to content

[feat](Nereids): Optimize query by pushing down aggregation through join on foreign key #36035

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
keanji-x merged 10 commits into from
Jul 1, 2024
Merged

[feat](Nereids): Optimize query by pushing down aggregation through join on foreign key #36035

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
7fe960d
push agg throght foreign key
keanji-x Jun 7, 2024
be9a002
add push down agg through join with foreign key
keanji-x Jun 7, 2024
00dbd2b
change rule type
keanji-x Jun 17, 2024
2c7d9f2
fix agg func
keanji-x Jun 18, 2024
fa120c1
process complex join
keanji-x Jun 21, 2024
e70a6e4
process complex join
keanji-x Jun 21, 2024
6b3a67b
add comment
keanji-x Jun 27, 2024
7ddf8dd
remove inner join check
keanji-x Jun 27, 2024
08c0b69
skip primary slot when constructing agg output
keanji-x Jun 27, 2024
7fe6542
Merge branch 'master' into add_agg_push_foreign
keanji-x Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOnPkFk;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOneSide;
import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject;
Expand Down Expand Up @@ -348,8 +349,9 @@ public class Rewriter extends AbstractBatchJobExecutor {
),

// this rule should be invoked after topic "Join pull up"
topic("eliminate group by keys according to fd items",
topDown(new EliminateGroupByKey())
topic("eliminate Aggregate according to fd items",
topDown(new EliminateGroupByKey()),
topDown(new PushDownAggThroughJoinOnPkFk())
),

topic("Limit optimization",
Expand Down
19 changes: 19 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ public int hashCode() {
}

private final Set<FuncDepsItem> items;
// determinants -> dependencies
private final Map<Set<Slot>, Set<Set<Slot>>> edges;

public FuncDeps() {
Expand Down Expand Up @@ -159,6 +160,24 @@ public boolean isFuncDeps(Set<Slot> dominate, Set<Slot> dependency) {
return items.contains(new FuncDepsItem(dominate, dependency));
}

public boolean isCircleDeps(Set<Slot> dominate, Set<Slot> dependency) {
return items.contains(new FuncDepsItem(dominate, dependency))
&& items.contains(new FuncDepsItem(dependency, dominate));
}

/**
* find the determinants of dependencies
*/
public Set<Set<Slot>> findDeterminats(Set<Slot> dependency) {
Set<Set<Slot>> determinants = new HashSet<>();
for (FuncDepsItem item : items) {
if (item.dependencies.equals(dependency)) {
determinants.add(item.determinants);
}
}
return determinants;
}

@Override
public String toString() {
return items.toString();
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ public enum RuleType {

PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE(RuleTypeClass.REWRITE),
PUSH_DOWN_AGG_THROUGH_JOIN(RuleTypeClass.REWRITE),

PUSH_DOWN_AGG_THROUGH_JOIN_ON_PKFK(RuleTypeClass.REWRITE),
TRANSPOSE_LOGICAL_SEMI_JOIN_LOGICAL_JOIN(RuleTypeClass.REWRITE),
TRANSPOSE_LOGICAL_SEMI_JOIN_LOGICAL_JOIN_PROJECT(RuleTypeClass.REWRITE),
LOGICAL_SEMI_JOIN_COMMUTE(RuleTypeClass.REWRITE),
Expand Down
Loading