From 24dbbd8a0d7b1a431903b13f6197b39b977d9495 Mon Sep 17 00:00:00 2001 From: jackwener Date: Mon, 16 May 2022 19:06:45 +0800 Subject: [PATCH] [feature](nereids): add join rules base code --- .../apache/doris/nereids/rules/RuleSet.java | 8 +-- .../apache/doris/nereids/rules/RuleType.java | 4 +- .../{ => join}/JoinCommutative.java | 33 ++++++++--- .../rules/exploration/join/JoinExchange.java | 57 +++++++++++++++++++ .../rules/exploration/join/JoinLAsscom.java | 54 ++++++++++++++++++ .../JoinLeftAssociative.java} | 14 ++++- .../doris/nereids/trees/plans/JoinType.java | 29 ++++++++++ .../plans/logical/JoinReorderContext.java | 51 +++++++++++++++++ .../trees/plans/logical/LogicalJoin.java | 7 +++ 9 files changed, 240 insertions(+), 17 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/{ => join}/JoinCommutative.java (59%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscom.java rename fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/{JoinAssociativeLeftToRight.java => join/JoinLeftAssociative.java} (77%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/JoinReorderContext.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 5d039a0cb1f73b..441ea80e2b3bbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -18,8 +18,8 @@ package org.apache.doris.nereids.rules; import org.apache.doris.nereids.rules.analysis.AnalysisUnboundRelation; -import org.apache.doris.nereids.rules.exploration.JoinAssociativeLeftToRight; -import org.apache.doris.nereids.rules.exploration.JoinCommutative; +import org.apache.doris.nereids.rules.exploration.join.JoinCommutative; +import org.apache.doris.nereids.rules.exploration.join.JoinLeftAssociative; import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Expression; @@ -39,8 +39,8 @@ public class RuleSet { .build(); public static final List> EXPLORATION_RULES = planRuleFactories() - .add(new JoinCommutative()) - .add(new JoinAssociativeLeftToRight()) + .add(new JoinCommutative(false)) + .add(new JoinLeftAssociative()) .build(); public static final List> IMPLEMENTATION_RULES = planRuleFactories() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index fee8a666e25ca2..700b97d1c5c978 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -29,7 +29,9 @@ public enum RuleType { // exploration rules LOGICAL_JOIN_COMMUTATIVE, - LOGICAL_JOIN_ASSOCIATIVE_LEFT_TO_RIGHT, + LOGICAL_LEFT_JOIN_ASSOCIATIVE, + LOGICAL_JOIN_L_ASSCOM, + LOGICAL_JOIN_EXCHANGE, // implementation rules LOGICAL_JOIN_TO_HASH_JOIN_RULE, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/JoinCommutative.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommutative.java similarity index 59% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/JoinCommutative.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommutative.java index 4f582487f98ce3..b7c3071c863e05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/JoinCommutative.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommutative.java @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.exploration; +package org.apache.doris.nereids.rules.exploration.join; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.JoinType; +import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -27,16 +27,31 @@ * rule factory for exchange inner join's children. */ public class JoinCommutative extends OneExplorationRuleFactory { + private boolean justApplyInnerOuterCrossJoin = false; + + private final SwapType swapType; + + /** + * If param is true, just apply rule in inner/full-outer/cross join. + */ + public JoinCommutative(boolean justApplyInnerOuterCrossJoin) { + this.justApplyInnerOuterCrossJoin = justApplyInnerOuterCrossJoin; + this.swapType = SwapType.ALL; + } + + public JoinCommutative(boolean justApplyInnerOuterCrossJoin, SwapType swapType) { + this.justApplyInnerOuterCrossJoin = justApplyInnerOuterCrossJoin; + this.swapType = swapType; + } + + enum SwapType { + BOTTOM_JOIN, ZIG_ZAG, ALL + } + @Override public Rule build() { return innerLogicalJoin().then(join -> { - // fixme, just for example now - return new LogicalJoin( - JoinType.INNER_JOIN, - join.getOnClause(), - join.right(), - join.left() - ); + return new LogicalJoin(join.getJoinType().swap(), join.getOnClause(), join.right(), join.left()); }).toRule(RuleType.LOGICAL_JOIN_COMMUTATIVE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java new file mode 100644 index 00000000000000..fe5cdc1b6bc20e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.join; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; + + +/** + * Rule for busy-tree, exchange the children node. + */ +public class JoinExchange extends OneExplorationRuleFactory { + /* + * topJoin newTopJoin + * / \ / \ + * leftJoin rightJoin --> newLeftJoin newRightJoin + * / \ / \ / \ / \ + * A B C D A C B D + */ + @Override + public Rule build() { + return innerLogicalJoin(innerLogicalJoin(), innerLogicalJoin()).then(topJoin -> { + LogicalJoin leftJoin = topJoin.left(); + LogicalJoin rightJoin = topJoin.left(); + + Plan a = leftJoin.left(); + Plan b = leftJoin.right(); + Plan c = rightJoin.left(); + Plan d = rightJoin.right(); + + LogicalJoin newLeftJoin = new LogicalJoin(leftJoin.getJoinType(), leftJoin.getOnClause(), a, c); + LogicalJoin newRightJoin = new LogicalJoin(rightJoin.getJoinType(), rightJoin.getOnClause(), b, d); + LogicalJoin newTopJoin = + new LogicalJoin(topJoin.getJoinType(), topJoin.getOnClause(), newLeftJoin, newRightJoin); + + return newTopJoin; + }).toRule(RuleType.LOGICAL_JOIN_EXCHANGE); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscom.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscom.java new file mode 100644 index 00000000000000..7552e9465335e6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLAsscom.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.join; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; + + +/** + * Rule for change inner join left associative to right. + */ +public class JoinLAsscom extends OneExplorationRuleFactory { + /* + * topJoin newTopJoin + * / \ / \ + * bottomJoin C --> newBottomJoin B + * / \ / \ + * A B A C + */ + @Override + public Rule build() { + return innerLogicalJoin(innerLogicalJoin(), any()).then(topJoin -> { + LogicalJoin bottomJoin = topJoin.left(); + + Plan c = topJoin.right(); + Plan a = bottomJoin.left(); + Plan b = bottomJoin.right(); + + LogicalJoin newBottomJoin = + new LogicalJoin(bottomJoin.getJoinType(), bottomJoin.getOnClause(), a, c); + LogicalJoin newTopJoin = + new LogicalJoin(bottomJoin.getJoinType(), topJoin.getOnClause(), newBottomJoin, b); + return newTopJoin; + }).toRule(RuleType.LOGICAL_JOIN_L_ASSCOM); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/JoinAssociativeLeftToRight.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLeftAssociative.java similarity index 77% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/JoinAssociativeLeftToRight.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLeftAssociative.java index 07085b58656865..214b0336a9916a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/JoinAssociativeLeftToRight.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinLeftAssociative.java @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.nereids.rules.exploration; +package org.apache.doris.nereids.rules.exploration.join; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.OneExplorationRuleFactory; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -26,7 +27,14 @@ /** * Rule factory for change inner join left associative to right. */ -public class JoinAssociativeLeftToRight extends OneExplorationRuleFactory { +public class JoinLeftAssociative extends OneExplorationRuleFactory { + /* + * topJoin newTopJoin + * / \ / \ + * bottomJoin C --> A newBottomJoin + * / \ / \ + * A B B C + */ @Override public Rule build() { return innerLogicalJoin(innerLogicalJoin(), any()).then(root -> { @@ -42,6 +50,6 @@ public Rule build() { root.right() ) ); - }).toRule(RuleType.LOGICAL_JOIN_ASSOCIATIVE_LEFT_TO_RIGHT); + }).toRule(RuleType.LOGICAL_LEFT_JOIN_ASSOCIATIVE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java index 858715d2c5d622..ab75a5060095d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/JoinType.java @@ -20,6 +20,10 @@ import org.apache.doris.analysis.JoinOperator; import org.apache.doris.common.AnalysisException; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + /** * All job type in Nereids. */ @@ -35,6 +39,19 @@ public enum JoinType { CROSS_JOIN, ; + private static final Map joinSwapMap = ImmutableMap + .builder() + .put(INNER_JOIN, INNER_JOIN) + .put(CROSS_JOIN, CROSS_JOIN) + .put(FULL_OUTER_JOIN, FULL_OUTER_JOIN) + .put(LEFT_SEMI_JOIN, RIGHT_SEMI_JOIN) + .put(RIGHT_SEMI_JOIN, LEFT_SEMI_JOIN) + .put(LEFT_OUTER_JOIN, RIGHT_OUTER_JOIN) + .put(RIGHT_OUTER_JOIN, LEFT_OUTER_JOIN) + .put(LEFT_ANTI_JOIN, RIGHT_ANTI_JOIN) + .put(RIGHT_ANTI_JOIN, LEFT_ANTI_JOIN) + .build(); + /** * Convert join type in Nereids to legacy join type in Doris. * @@ -66,4 +83,16 @@ public static JoinOperator toJoinOperator(JoinType joinType) throws AnalysisExce throw new AnalysisException("Not support join operator: " + joinType.name()); } } + + public final boolean isInnerOrOuterOrCrossJoin() { + return this == INNER_JOIN || this == CROSS_JOIN || this == FULL_OUTER_JOIN; + } + + public final boolean isSwapJoinType() { + return joinSwapMap.containsKey(this); + } + + public JoinType swap() { + return joinSwapMap.get(this); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/JoinReorderContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/JoinReorderContext.java new file mode 100644 index 00000000000000..c56da73b4cf3bf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/JoinReorderContext.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + + +/** + * JoinReorderContext for Duplicate free. + * Paper: Improving Join Reorderability with Compensation Operators + */ +public class JoinReorderContext { + // left deep tree + private boolean hasCommute = false; + private boolean hasTopPushThrough = false; + + public JoinReorderContext() { + } + + void copyFrom(JoinReorderContext joinReorderContext) { + this.hasCommute = joinReorderContext.hasCommute; + this.hasTopPushThrough = joinReorderContext.hasTopPushThrough; + } + + JoinReorderContext copy() { + JoinReorderContext joinReorderContext = new JoinReorderContext(); + joinReorderContext.copyFrom(this); + return joinReorderContext; + } + + public boolean isHasCommute() { + return hasCommute; + } + + public boolean isHasTopPushThrough() { + return hasTopPushThrough; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index 04b23ea76fb78d..fd4783c3caefbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -40,6 +40,9 @@ public class LogicalJoin< private final JoinType joinType; private final Expression onClause; + // Use for top-to-down join reorder + private final JoinReorderContext joinReorderContext = new JoinReorderContext(); + /** * Constructor for LogicalJoinPlan. * @@ -92,4 +95,8 @@ public String toString() { } return sb.append(")").toString(); } + + public JoinReorderContext getJoinReorderContext() { + return joinReorderContext; + } }