From b2dad10671a06873166973c1400ba415adca55f6 Mon Sep 17 00:00:00 2001 From: jianghaochen Date: Fri, 6 Jan 2023 11:33:41 +0800 Subject: [PATCH] [Feature](Nereids) support MarkJoin --- .../apache/doris/analysis/StmtRewriter.java | 14 + .../doris/nereids/StatementContext.java | 25 ++ .../glue/translator/ExpressionTranslator.java | 7 + .../translator/PhysicalPlanTranslator.java | 45 ++- .../translator/PlanTranslatorContext.java | 11 + .../nereids/jobs/batch/NereidsRewriter.java | 9 +- .../joinorder/hypergraph/GraphSimplifier.java | 2 + .../hypergraph/receiver/PlanReceiver.java | 31 +- .../nereids/parser/LogicalPlanBuilder.java | 2 + .../post/RuntimeFilterGenerator.java | 2 +- .../nereids/processor/post/Validator.java | 4 +- .../rules/analysis/BindExpression.java | 7 +- .../rules/analysis/CheckAfterRewrite.java | 7 +- .../rules/analysis/ColumnAliasGenerator.java | 39 ++ .../rules/analysis/FunctionBinder.java | 4 +- .../nereids/rules/analysis/SlotBinder.java | 14 +- .../rules/analysis/SubExprAnalyzer.java | 28 ++ .../rules/analysis/SubqueryToApply.java | 343 ++++++++++++++++-- .../exploration/join/InnerJoinLAsscom.java | 4 +- .../join/InnerJoinLAsscomProject.java | 1 + .../join/InnerJoinLeftAssociate.java | 1 + .../join/InnerJoinRightAssociate.java | 1 + .../rules/exploration/join/JoinCommute.java | 1 + .../rules/exploration/join/JoinExchange.java | 1 + .../exploration/join/OuterJoinAssoc.java | 1 + .../join/OuterJoinAssocProject.java | 1 + .../exploration/join/OuterJoinLAsscom.java | 1 + .../join/OuterJoinLAsscomProject.java | 1 + .../join/SemiJoinLogicalJoinTranspose.java | 1 + .../SemiJoinLogicalJoinTransposeProject.java | 1 + .../join/SemiJoinSemiJoinTranspose.java | 1 + .../SemiJoinSemiJoinTransposeProject.java | 1 + .../expression/rewrite/ExpressionRewrite.java | 3 +- .../implementation/LogicalJoinToHashJoin.java | 1 + .../LogicalJoinToNestedLoopJoin.java | 1 + .../rewrite/logical/ExistsApplyToJoin.java | 34 +- .../logical/ExtractFilterFromCrossJoin.java | 2 +- .../logical/FindHashConditionForJoin.java | 1 + .../rules/rewrite/logical/InApplyToJoin.java | 18 +- ...latedFilterUnderApplyAggregateProject.java | 3 +- .../logical/PullUpProjectUnderApply.java | 3 +- .../rewrite/logical/PushFilterInsideJoin.java | 4 +- .../logical/PushdownFilterThroughJoin.java | 1 + .../logical/PushdownJoinOtherCondition.java | 2 +- .../rules/rewrite/logical/ReorderJoin.java | 74 +++- .../rewrite/logical/ScalarApplyToJoin.java | 20 +- .../UnCorrelatedApplyAggregateFilter.java | 2 + .../logical/UnCorrelatedApplyFilter.java | 3 +- .../UnCorrelatedApplyProjectFilter.java | 3 +- .../nereids/stats/ExpressionEstimation.java | 7 + .../expressions/AssertNumRowsElement.java | 4 +- .../expressions/MarkJoinSlotReference.java | 64 ++++ .../expressions/SlotNotFromChildren.java | 26 ++ .../expressions/VirtualSlotReference.java | 2 +- .../visitor/ExpressionVisitor.java | 5 + .../nereids/trees/plans/algebra/Join.java | 7 + .../trees/plans/logical/LogicalApply.java | 47 ++- .../trees/plans/logical/LogicalJoin.java | 71 +++- .../trees/plans/logical/UsingJoin.java | 24 +- .../plans/physical/AbstractPhysicalJoin.java | 19 +- .../plans/physical/PhysicalHashJoin.java | 22 +- .../physical/PhysicalNestedLoopJoin.java | 26 +- .../org/apache/doris/nereids/util/Utils.java | 5 + .../apache/doris/planner/HashJoinNode.java | 8 +- .../apache/doris/planner/JoinNodeBase.java | 8 +- .../doris/planner/NestedLoopJoinNode.java | 8 +- .../ChildOutputPropertyDeriverTest.java | 9 +- .../RequestPropertyDeriverTest.java | 7 +- .../analysis/AnalyzeWhereSubqueryTest.java | 38 +- .../logical/FindHashConditionForJoinTest.java | 3 +- .../PushdownJoinOtherConditionTest.java | 7 +- .../nereids/trees/plans/MarkJoinTest.java | 249 +++++++++++++ .../nereids/trees/plans/PlanEqualsTest.java | 6 +- .../nereids/trees/plans/PlanToStringTest.java | 2 +- .../nereids/util/LogicalPlanBuilder.java | 3 +- .../sub_query_correlated.out | 147 ++++++++ .../sub_query_correlated.groovy | 105 +++++- .../sub_query_diff_old_optimize.groovy | 8 + 78 files changed, 1531 insertions(+), 192 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotNotFromChildren.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/MarkJoinTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java index fd8a0710f7f854..5b4fce1f0f9470 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java @@ -504,6 +504,10 @@ private static void processOneSubquery(SelectStmt stmt, + "expression: " + exprWithSubquery.toSql()); } + if (exprWithSubquery instanceof BinaryPredicate && (childrenContainInOrExists(exprWithSubquery))) { + throw new AnalysisException("Not support binaryOperator children at least one is in or exists subquery" + + exprWithSubquery.toSql()); + } if (exprWithSubquery instanceof ExistsPredicate) { // Check if we can determine the result of an ExistsPredicate during analysis. @@ -542,6 +546,16 @@ private static void processOneSubquery(SelectStmt stmt, } } + private static boolean childrenContainInOrExists(Expr expr) { + boolean contain = false; + for (Expr child : expr.getChildren()) { + contain = contain || child instanceof InPredicate || child instanceof ExistsPredicate; + if (contain) { + break; + } + } + return contain; + } /** * Replace an ExistsPredicate that contains a subquery with a BoolLiteral if we diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 53e8ff4b165f81..51a62d4ad4f263 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.common.IdGenerator; +import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.qe.ConnectContext; @@ -28,7 +29,9 @@ import com.google.common.base.Suppliers; import com.google.common.collect.Maps; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import javax.annotation.concurrent.GuardedBy; /** @@ -51,6 +54,10 @@ public class StatementContext { private StatementBase parsedStatement; + private Set columnNames; + + private ColumnAliasGenerator columnAliasGenerator; + public StatementContext() { this.connectContext = ConnectContext.get(); } @@ -111,4 +118,22 @@ public synchronized T getOrRegisterCache(String key, Supplier cacheSuppli } return supplier.get(); } + + public Set getColumnNames() { + return columnNames == null ? new HashSet<>() : columnNames; + } + + public void setColumnNames(Set columnNames) { + this.columnNames = columnNames; + } + + public ColumnAliasGenerator getColumnAliasGenerator() { + return columnAliasGenerator == null + ? columnAliasGenerator = new ColumnAliasGenerator(this) + : columnAliasGenerator; + } + + public String generateColumnName() { + return getColumnAliasGenerator().getNextAlias(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java index c379e49b8f8a94..3db32de4391e14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java @@ -57,6 +57,7 @@ import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.NullSafeEqual; import org.apache.doris.nereids.trees.expressions.Or; @@ -193,6 +194,12 @@ public Expr visitSlotReference(SlotReference slotReference, PlanTranslatorContex return context.findSlotRef(slotReference.getExprId()); } + @Override + public Expr visitMarkJoinReference(MarkJoinSlotReference markJoinSlotReference, PlanTranslatorContext context) { + return markJoinSlotReference.isExistsHasAgg() + ? new BoolLiteral(true) : context.findSlotRef(markJoinSlotReference.getExprId()); + } + @Override public Expr visitLiteral(Literal literal, PlanTranslatorContext context) { return literal.toLegacyLiteral(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 500f0360a5e7f1..74c9517aff1f3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -60,6 +60,7 @@ import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -956,7 +957,7 @@ public PlanFragment visitPhysicalHashJoin( HashJoinNode hashJoinNode = new HashJoinNode(context.nextPlanNodeId(), leftPlanRoot, rightPlanRoot, JoinType.toJoinOperator(joinType), execEqConjuncts, Lists.newArrayList(), - null, null, null); + null, null, null, hashJoin.isMarkJoin()); PlanFragment currentFragment; if (JoinUtils.shouldColocateJoin(physicalHashJoin)) { @@ -1012,13 +1013,15 @@ public PlanFragment visitPhysicalHashJoin( .forEach(s -> hashOutputSlotReferenceMap.put(s.getExprId(), s)); Map leftChildOutputMap = Maps.newHashMap(); - hashJoin.child(0).getOutput().stream() + Stream.concat(hashJoin.child(0).getOutput().stream(), hashJoin.child(0).getNonUserVisibleOutput().stream()) .map(SlotReference.class::cast) .forEach(s -> leftChildOutputMap.put(s.getExprId(), s)); + context.getOutputMarkJoinSlot().stream().forEach(s -> leftChildOutputMap.put(s.getExprId(), s)); Map rightChildOutputMap = Maps.newHashMap(); - hashJoin.child(1).getOutput().stream() + Stream.concat(hashJoin.child(1).getOutput().stream(), hashJoin.child(1).getNonUserVisibleOutput().stream()) .map(SlotReference.class::cast) .forEach(s -> rightChildOutputMap.put(s.getExprId(), s)); + context.getOutputMarkJoinSlot().stream().forEach(s -> rightChildOutputMap.put(s.getExprId(), s)); // translate runtime filter context.getRuntimeTranslator().ifPresent(runtimeFilterTranslator -> runtimeFilterTranslator .getRuntimeFilterOfHashJoinNode(physicalHashJoin) @@ -1040,6 +1043,9 @@ public PlanFragment visitPhysicalHashJoin( SlotReference sf = leftChildOutputMap.get(context.findExprId(leftSlotDescriptor.getId())); SlotDescriptor sd = context.createSlotDesc(intermediateDescriptor, sf); leftIntermediateSlotDescriptor.add(sd); + if (sf instanceof MarkJoinSlotReference && hashJoin.getFilterConjuncts().isEmpty()) { + outputSlotReferences.add(sf); + } } } else if (hashJoin.getOtherJoinConjuncts().isEmpty() && (joinType == JoinType.RIGHT_ANTI_JOIN || joinType == JoinType.RIGHT_SEMI_JOIN)) { @@ -1076,6 +1082,14 @@ public PlanFragment visitPhysicalHashJoin( } } + if (hashJoin.getMarkJoinSlotReference().isPresent()) { + if (hashJoin.getFilterConjuncts().isEmpty()) { + outputSlotReferences.add(hashJoin.getMarkJoinSlotReference().get()); + context.setOutputMarkJoinSlot(hashJoin.getMarkJoinSlotReference().get()); + } + context.createSlotDesc(intermediateDescriptor, hashJoin.getMarkJoinSlotReference().get()); + } + // set slots as nullable for outer join if (joinType == JoinType.LEFT_OUTER_JOIN || joinType == JoinType.FULL_OUTER_JOIN) { rightIntermediateSlotDescriptor.forEach(sd -> sd.setIsNullable(true)); @@ -1142,7 +1156,7 @@ public PlanFragment visitPhysicalNestedLoopJoin( NestedLoopJoinNode nestedLoopJoinNode = new NestedLoopJoinNode(context.nextPlanNodeId(), leftFragmentPlanRoot, rightFragmentPlanRoot, tupleIds, JoinType.toJoinOperator(joinType), - null, null, null); + null, null, null, nestedLoopJoin.isMarkJoin()); if (nestedLoopJoin.getStats() != null) { nestedLoopJoinNode.setCardinality((long) nestedLoopJoin.getStats().getRowCount()); } @@ -1157,13 +1171,17 @@ public PlanFragment visitPhysicalNestedLoopJoin( } Map leftChildOutputMap = Maps.newHashMap(); - nestedLoopJoin.child(0).getOutput().stream() + Stream.concat(nestedLoopJoin.child(0).getOutput().stream(), + nestedLoopJoin.child(0).getNonUserVisibleOutput().stream()) .map(SlotReference.class::cast) .forEach(s -> leftChildOutputMap.put(s.getExprId(), s)); + context.getOutputMarkJoinSlot().stream().forEach(s -> leftChildOutputMap.put(s.getExprId(), s)); Map rightChildOutputMap = Maps.newHashMap(); - nestedLoopJoin.child(1).getOutput().stream() + Stream.concat(nestedLoopJoin.child(1).getOutput().stream(), + nestedLoopJoin.child(1).getNonUserVisibleOutput().stream()) .map(SlotReference.class::cast) .forEach(s -> rightChildOutputMap.put(s.getExprId(), s)); + context.getOutputMarkJoinSlot().stream().forEach(s -> rightChildOutputMap.put(s.getExprId(), s)); // make intermediate tuple List leftIntermediateSlotDescriptor = Lists.newArrayList(); List rightIntermediateSlotDescriptor = Lists.newArrayList(); @@ -1198,6 +1216,7 @@ public PlanFragment visitPhysicalNestedLoopJoin( .map(outputSlotReferenceMap::get) .filter(Objects::nonNull) .collect(Collectors.toList()); + // TODO: because of the limitation of be, the VNestedLoopJoinNode will output column from both children // in the intermediate tuple, so fe have to do the same, if be fix the problem, we can change it back. for (SlotDescriptor leftSlotDescriptor : leftSlotDescriptors) { @@ -1207,6 +1226,9 @@ public PlanFragment visitPhysicalNestedLoopJoin( SlotReference sf = leftChildOutputMap.get(context.findExprId(leftSlotDescriptor.getId())); SlotDescriptor sd = context.createSlotDesc(intermediateDescriptor, sf); leftIntermediateSlotDescriptor.add(sd); + if (sf instanceof MarkJoinSlotReference && nestedLoopJoin.getFilterConjuncts().isEmpty()) { + outputSlotReferences.add(sf); + } } for (SlotDescriptor rightSlotDescriptor : rightSlotDescriptors) { if (!rightSlotDescriptor.isMaterialized()) { @@ -1215,6 +1237,17 @@ public PlanFragment visitPhysicalNestedLoopJoin( SlotReference sf = rightChildOutputMap.get(context.findExprId(rightSlotDescriptor.getId())); SlotDescriptor sd = context.createSlotDesc(intermediateDescriptor, sf); rightIntermediateSlotDescriptor.add(sd); + if (sf instanceof MarkJoinSlotReference && nestedLoopJoin.getFilterConjuncts().isEmpty()) { + outputSlotReferences.add(sf); + } + } + + if (nestedLoopJoin.getMarkJoinSlotReference().isPresent()) { + if (nestedLoopJoin.getFilterConjuncts().isEmpty()) { + outputSlotReferences.add(nestedLoopJoin.getMarkJoinSlotReference().get()); + context.setOutputMarkJoinSlot(nestedLoopJoin.getMarkJoinSlotReference().get()); + } + context.createSlotDesc(intermediateDescriptor, nestedLoopJoin.getMarkJoinSlotReference().get()); } // set slots as nullable for outer join diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index da85dceb7961ab..88441b759b535b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -29,6 +29,7 @@ import org.apache.doris.common.IdGenerator; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; @@ -81,6 +82,8 @@ public class PlanTranslatorContext { private final Map bufferedSlotRefForWindow = Maps.newHashMap(); private TupleDescriptor bufferedTupleForWindow = null; + private List outputMarkJoinSlot = Lists.newArrayList(); + public PlanTranslatorContext(CascadesContext ctx) { this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext()); } @@ -210,4 +213,12 @@ public TupleDescriptor getTupleDesc(TupleId tupleId) { public DescriptorTable getDescTable() { return descTable; } + + public void setOutputMarkJoinSlot(MarkJoinSlotReference markJoinSlotReference) { + outputMarkJoinSlot.add(markJoinSlotReference); + } + + public List getOutputMarkJoinSlot() { + return outputMarkJoinSlot; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java index 1d51420a7edbcf..c9ad180ba98799 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java @@ -84,7 +84,6 @@ public class NereidsRewriter extends BatchRewriteJob { new AvgDistinctToSumDivCount(), new CountDistinctRewrite(), - new NormalizeAggregate(), new ExtractFilterFromCrossJoin() ), @@ -116,6 +115,14 @@ public class NereidsRewriter extends BatchRewriteJob { ) ), + // The rule modification needs to be done after the subquery is unnested, + // because for scalarSubQuery, the connection condition is stored in apply in the analyzer phase, + // but when normalizeAggregate is performed, the members in apply cannot be obtained, + // resulting in inconsistent output results and results in apply + topDown( + new NormalizeAggregate() + ), + topDown( new AdjustAggregateNullableForEmptySet() ), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java index a0817daf9db83a..db64127bf329e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/GraphSimplifier.java @@ -432,6 +432,7 @@ private double calCost(Edge edge, StatsDeriveResult stats, join.getJoinType(), join.getHashJoinConjuncts(), join.getOtherJoinConjuncts(), + join.getMarkJoinSlotReference(), join.getLogicalProperties(), join.left(), join.right()); @@ -442,6 +443,7 @@ private double calCost(Edge edge, StatsDeriveResult stats, join.getHashJoinConjuncts(), join.getOtherJoinConjuncts(), join.getHint(), + join.getMarkJoinSlotReference(), join.getLogicalProperties(), join.left(), join.right()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java index 4e38346e6b653d..1079f4a6d2f1b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java @@ -57,6 +57,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -226,16 +227,19 @@ private List proposeAllPhysicalJoins(JoinType joinType, Plan left, Plan ri () -> JoinUtils.getJoinOutput(joinType, left, right)); if (JoinUtils.shouldNestedLoopJoin(joinType, hashConjuncts)) { return Lists.newArrayList( - new PhysicalNestedLoopJoin<>(joinType, hashConjuncts, otherConjuncts, joinProperties, left, - right), - new PhysicalNestedLoopJoin<>(joinType.swap(), hashConjuncts, otherConjuncts, joinProperties, + new PhysicalNestedLoopJoin<>(joinType, hashConjuncts, otherConjuncts, + Optional.empty(), joinProperties, + left, right), + new PhysicalNestedLoopJoin<>(joinType.swap(), hashConjuncts, otherConjuncts, Optional.empty(), + joinProperties, right, left)); } else { return Lists.newArrayList( - new PhysicalHashJoin<>(joinType, hashConjuncts, otherConjuncts, JoinHint.NONE, joinProperties, - left, - right), + new PhysicalHashJoin<>(joinType, hashConjuncts, otherConjuncts, JoinHint.NONE, Optional.empty(), + joinProperties, + left, right), new PhysicalHashJoin<>(joinType.swap(), hashConjuncts, otherConjuncts, JoinHint.NONE, + Optional.empty(), joinProperties, right, left)); } @@ -258,6 +262,17 @@ private JoinType extractJoinTypeAndConjuncts(List edges, List return joinType; } + private boolean extractIsMarkJoin(List edges) { + boolean isMarkJoin = false; + JoinType joinType = null; + for (Edge edge : edges) { + Preconditions.checkArgument(joinType == null || joinType == edge.getJoinType()); + isMarkJoin = edge.getJoin().isMarkJoin() || isMarkJoin; + joinType = edge.getJoinType(); + } + return isMarkJoin; + } + @Override public void addGroup(long bitmap, Group group) { Preconditions.checkArgument(LongBitmap.getCardinality(bitmap) == 1); @@ -322,8 +337,8 @@ private void makeLogicalExpression(Group root) { } else if (physicalPlan instanceof AbstractPhysicalJoin) { AbstractPhysicalJoin physicalJoin = (AbstractPhysicalJoin) physicalPlan; logicalPlan = new LogicalJoin<>(physicalJoin.getJoinType(), physicalJoin.getHashJoinConjuncts(), - physicalJoin.getOtherJoinConjuncts(), JoinHint.NONE, physicalJoin.child(0), - physicalJoin.child(1)); + physicalJoin.getOtherJoinConjuncts(), JoinHint.NONE, physicalJoin.getMarkJoinSlotReference(), + physicalJoin.child(0), physicalJoin.child(1)); } else { throw new RuntimeException("DPhyp can only handle join and project operator"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 3947e8139b3cc6..60685d7540664c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -1218,6 +1218,7 @@ public LogicalPlan visitFromClause(FromClauseContext ctx) { ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, + Optional.empty(), left, right); left = withJoinRelations(left, relation); @@ -1481,6 +1482,7 @@ private LogicalPlan withJoinRelations(LogicalPlan input, RelationContext ctx) { condition.map(ExpressionUtils::extractConjunction) .orElse(ExpressionUtils.EMPTY_CONDITION), joinHint, + Optional.empty(), last, plan(join.relationPrimary())); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java index 7c8658fc3736f5..2629d93c749eb5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java @@ -80,7 +80,7 @@ public PhysicalPlan visitPhysicalHashJoin(PhysicalHashJoin> aliasTransferMap = ctx.getAliasTransferMap(); join.right().accept(this, context); join.left().accept(this, context); - if (deniedJoinType.contains(join.getJoinType())) { + if (deniedJoinType.contains(join.getJoinType()) || join.isMarkJoin()) { // copy to avoid bug when next call of getOutputSet() Set slots = join.getOutputSet(); slots.forEach(aliasTransferMap::remove); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java index 6125fcaad4269d..87252a7e5ce86d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/Validator.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; @@ -78,7 +79,8 @@ public Plan visitPhysicalFilter(PhysicalFilter filter, CascadesC .>map(expr -> expr.collect(Slot.class::isInstance)) .flatMap(Collection::stream).collect(Collectors.toSet()); for (Slot slot : slotsUsedByFilter) { - Preconditions.checkState(childOutputSet.contains(slot)); + Preconditions.checkState(childOutputSet.contains(slot) + || slot instanceof MarkJoinSlotReference); } child.accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 6af0d87e3fb140..701e18c9d111c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -153,8 +153,8 @@ protected boolean condition(Rule rule, Plan plan) { LogicalJoin lj = new LogicalJoin<>(using.getJoinType() == JoinType.CROSS_JOIN ? JoinType.INNER_JOIN : using.getJoinType(), using.getHashJoinConjuncts(), - using.getOtherJoinConjuncts(), using.getHint(), using.left(), - using.right()); + using.getOtherJoinConjuncts(), using.getHint(), using.getMarkJoinSlotReference(), + using.left(), using.right()); List unboundSlots = lj.getHashJoinConjuncts(); Set slotNames = new HashSet<>(); List leftOutput = new ArrayList<>(lj.left().getOutput()); @@ -201,7 +201,8 @@ protected boolean condition(Rule rule, Plan plan) { .map(expr -> bindFunction(expr, ctx.cascadesContext)) .collect(Collectors.toList()); return new LogicalJoin<>(join.getJoinType(), - hashJoinConjuncts, cond, join.getHint(), join.left(), join.right()); + hashJoinConjuncts, cond, join.getHint(), join.getMarkJoinSlotReference(), + join.left(), join.right()); }) ), RuleType.BINDING_AGGREGATE_SLOT.build( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java index 3e8a94e5cf3816..0f1e50f1448055 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotNotFromChildren; import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.plans.Plan; @@ -71,8 +72,8 @@ private void checkAllSlotReferenceFromChildren(Plan plan) { } } - private Set removeValidSlotsNotFromChildren(Set virtualSlots, Set childrenOutput) { - return virtualSlots.stream() + private Set removeValidSlotsNotFromChildren(Set slots, Set childrenOutput) { + return slots.stream() .filter(expr -> { if (expr instanceof VirtualSlotReference) { List realExpressions = ((VirtualSlotReference) expr).getRealExpressions(); @@ -85,7 +86,7 @@ private Set removeValidSlotsNotFromChildren(Set virtualSlots, Set !childrenOutput.contains(realUsedExpr.getExprId())); } else { - return true; + return !(expr instanceof SlotNotFromChildren); } }) .collect(Collectors.toSet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java new file mode 100644 index 00000000000000..75e5e0cc5e15ab --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ColumnAliasGenerator.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/ColumnAliasGenerator.java +// and modified by Doris + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.common.AliasGenerator; +import org.apache.doris.nereids.StatementContext; + +import com.google.common.base.Preconditions; + +/** + * Generate the table name required in the rewrite process. + */ +public class ColumnAliasGenerator extends AliasGenerator { + private static final String DEFAULT_COL_ALIAS_PREFIX = "$c$"; + + public ColumnAliasGenerator(StatementContext statementContext) { + Preconditions.checkNotNull(statementContext); + aliasPrefix = DEFAULT_COL_ALIAS_PREFIX; + usedAliases.addAll(statementContext.getColumnNames()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FunctionBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FunctionBinder.java index 63ede1e1adcd77..6a7286a692eedf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FunctionBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FunctionBinder.java @@ -34,6 +34,7 @@ import org.apache.doris.nereids.trees.expressions.InPredicate; import org.apache.doris.nereids.trees.expressions.IntegralDivide; import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; import org.apache.doris.nereids.trees.expressions.WhenClause; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; @@ -187,7 +188,8 @@ public Expression visitCompoundPredicate(CompoundPredicate compoundPredicate, Ca Expression right = compoundPredicate.right().accept(this, context); Expression ret = compoundPredicate.withChildren(left, right); ret.children().forEach(e -> { - if (!e.getDataType().isBooleanType() && !e.getDataType().isNullType()) { + if (!e.getDataType().isBooleanType() && !e.getDataType().isNullType() + && !(e instanceof SubqueryExpr)) { throw new AnalysisException(String.format( "Operand '%s' part of predicate " + "'%s' should return type 'BOOLEAN' but " + "returns type '%s'.", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java index 0d1785eafb560a..a045cd0a195562 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SlotBinder.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import java.util.List; @@ -69,12 +70,15 @@ public Expression bind(Expression expression) { public Expression visitUnboundAlias(UnboundAlias unboundAlias, CascadesContext context) { Expression child = unboundAlias.child().accept(this, context); if (unboundAlias.getAlias().isPresent()) { + collectColumnNames(unboundAlias.getAlias().get()); return new Alias(child, unboundAlias.getAlias().get()); } if (child instanceof NamedExpression) { + collectColumnNames(((NamedExpression) child).getName()); return new Alias(child, ((NamedExpression) child).getName()); } else { // TODO: resolve aliases + collectColumnNames(child.toSql()); return new Alias(child, child.toSql()); } } @@ -98,7 +102,8 @@ public Slot visitUnboundSlot(UnboundSlot unboundSlot, CascadesContext context) { // if unbound finally, check will throw exception return unboundSlot; case 1: - if (!foundInThisScope) { + if (!foundInThisScope + && !getScope().getOuterScope().get().getCorrelatedSlots().contains(bounded.get(0))) { getScope().getOuterScope().get().getCorrelatedSlots().add(bounded.get(0)); } return bounded.get(0); @@ -218,4 +223,11 @@ private List bindSlot(UnboundSlot unboundSlot, List boundSlots) { + StringUtils.join(nameParts, ".")); }).collect(Collectors.toList()); } + + private void collectColumnNames(String columnName) { + Preconditions.checkNotNull(getCascadesContext()); + if (!getCascadesContext().getStatementContext().getColumnNames().add(columnName)) { + throw new AnalysisException("Collect column name failed, columnName : " + columnName); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java index a51e049999212e..417498f7a10364 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubExprAnalyzer.java @@ -20,6 +20,8 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.BinaryOperator; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; @@ -81,6 +83,7 @@ public Expression visitInSubquery(InSubquery expr, CascadesContext context) { AnalyzedResult analyzedResult = analyzeSubquery(expr); checkOutputColumn(analyzedResult.getLogicalPlan()); + checkHasNotAgg(analyzedResult); checkHasGroupBy(analyzedResult); checkRootIsLimit(analyzedResult); @@ -101,6 +104,21 @@ public Expression visitScalarSubquery(ScalarSubquery scalar, CascadesContext con return new ScalarSubquery(analyzedResult.getLogicalPlan(), analyzedResult.getCorrelatedSlots()); } + @Override + public Expression visitBinaryOperator(BinaryOperator binaryOperator, CascadesContext context) { + if (childrenAtLeastOneInOrExistsSub(binaryOperator) && (binaryOperator instanceof ComparisonPredicate)) { + throw new AnalysisException("Not support binaryOperator children at least one is in or exists subquery"); + } + return visit(binaryOperator, context); + } + + private boolean childrenAtLeastOneInOrExistsSub(BinaryOperator binaryOperator) { + return binaryOperator.left().anyMatch(InSubquery.class::isInstance) + || binaryOperator.left().anyMatch(Exists.class::isInstance) + || binaryOperator.right().anyMatch(InSubquery.class::isInstance) + || binaryOperator.right().anyMatch(Exists.class::isInstance); + } + private void checkOutputColumn(LogicalPlan plan) { if (plan.getOutput().size() != 1) { throw new AnalysisException("Multiple columns returned by subquery are not yet supported. Found " @@ -129,6 +147,16 @@ private void checkHasGroupBy(AnalyzedResult analyzedResult) { } } + private void checkHasNotAgg(AnalyzedResult analyzedResult) { + if (!analyzedResult.isCorrelated()) { + return; + } + if (analyzedResult.hasAgg()) { + throw new AnalysisException("Unsupported correlated subquery with grouping and/or aggregation " + + analyzedResult.getLogicalPlan()); + } + } + private void checkRootIsLimit(AnalyzedResult analyzedResult) { if (!analyzedResult.isCorrelated()) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java index 995a095a345fab..7ca11befe93002 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java @@ -18,30 +18,43 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.BinaryOperator; import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Not; +import org.apache.doris.nereids.trees.expressions.Or; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; -import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Aggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -58,23 +71,55 @@ public List buildRules() { RuleType.FILTER_SUBQUERY_TO_APPLY.build( logicalFilter().thenApply(ctx -> { LogicalFilter filter = ctx.root; - Set subqueryExprs = filter.getPredicate().collect(SubqueryExpr.class::isInstance); - if (subqueryExprs.isEmpty()) { + + ImmutableList subqueryExprsList = filter.getConjuncts().stream() + .map(e -> (Set) e.collect(SubqueryExpr.class::isInstance)) + .collect(ImmutableList.toImmutableList()); + if (subqueryExprsList.stream() + .flatMap(Collection::stream).noneMatch(SubqueryExpr.class::isInstance)) { return filter; } - // first step: Replace the subquery of predicate in LogicalFilter - // second step: Replace subquery with LogicalApply - return new LogicalFilter<>(new ReplaceSubquery().replace(filter.getConjuncts()), - subqueryToApply( - subqueryExprs, filter.child(), ctx.cascadesContext - )); + List oldConjuncts = ImmutableList.copyOf(filter.getConjuncts()); + ImmutableList.Builder newConjuncts = new ImmutableList.Builder<>(); + LogicalPlan applyPlan = null; + LogicalPlan tmpPlan = (LogicalPlan) filter.child(); + + // Subquery traversal with the conjunct of and as the granularity. + for (int i = 0; i < subqueryExprsList.size(); ++i) { + Set subqueryExprs = subqueryExprsList.get(i); + if (subqueryExprs.isEmpty()) { + newConjuncts.add(oldConjuncts.get(i)); + continue; + } + + // first step: Replace the subquery of predicate in LogicalFilter + // second step: Replace subquery with LogicalApply + ReplaceSubquery replaceSubquery = new ReplaceSubquery( + ctx.statementContext, false); + SubqueryContext context = new SubqueryContext(subqueryExprs); + Expression conjunct = replaceSubquery.replace(oldConjuncts.get(i), context); + + applyPlan = subqueryToApply(subqueryExprs.stream() + .collect(ImmutableList.toImmutableList()), tmpPlan, + context.getSubqueryToMarkJoinSlot(), + context.getSubqueryCorrespondingConjunct(), ctx.cascadesContext, + Optional.of(conjunct), false); + tmpPlan = applyPlan; + if (!(subqueryExprs.size() == 1 + && subqueryExprs.stream().anyMatch(ScalarSubquery.class::isInstance))) { + newConjuncts.add(conjunct); + } + } + Set conjects = new HashSet<>(); + conjects.addAll(newConjuncts.build()); + return new LogicalFilter<>(conjects, applyPlan); }) ), RuleType.PROJECT_SUBQUERY_TO_APPLY.build( logicalProject().thenApply(ctx -> { LogicalProject project = ctx.root; - Set subqueryExprs = new HashSet<>(); + Set subqueryExprs = new LinkedHashSet<>(); project.getProjects().stream() .filter(Alias.class::isInstance) .map(Alias.class::cast) @@ -86,40 +131,97 @@ public List buildRules() { return project; } + SubqueryContext context = new SubqueryContext(subqueryExprs); return new LogicalProject(project.getProjects().stream() - .map(p -> p.withChildren(new ReplaceSubquery().replace(p))) + .map(p -> p.withChildren( + new ReplaceSubquery(ctx.statementContext, true) + .replace(p, context))) .collect(ImmutableList.toImmutableList()), subqueryToApply( - subqueryExprs, project.child(), ctx.cascadesContext + subqueryExprs.stream().collect(ImmutableList.toImmutableList()), + (LogicalPlan) project.child(), + context.getSubqueryToMarkJoinSlot(), context.getSubqueryCorrespondingConjunct(), + ctx.cascadesContext, + Optional.empty(), true )); }) ) ); } - private Plan subqueryToApply(Set subqueryExprs, - Plan childPlan, CascadesContext ctx) { - Plan tmpPlan = childPlan; - for (SubqueryExpr subqueryExpr : subqueryExprs) { + private LogicalPlan subqueryToApply(List subqueryExprs, LogicalPlan childPlan, + Map> subqueryToMarkJoinSlot, + Map subqueryCorrespondingConject, + CascadesContext ctx, + Optional conjunct, boolean isProject) { + LogicalPlan tmpPlan = childPlan; + for (int i = 0; i < subqueryExprs.size(); ++i) { + SubqueryExpr subqueryExpr = subqueryExprs.get(i); if (!ctx.subqueryIsAnalyzed(subqueryExpr)) { - tmpPlan = addApply(subqueryExpr, tmpPlan, ctx); + tmpPlan = addApply(subqueryExpr, tmpPlan, + subqueryToMarkJoinSlot, subqueryCorrespondingConject, ctx, conjunct, + isProject, subqueryExprs.size() == 1); } } return tmpPlan; } - private LogicalPlan addApply(SubqueryExpr subquery, Plan childPlan, CascadesContext ctx) { + private LogicalPlan addApply(SubqueryExpr subquery, LogicalPlan childPlan, + Map> subqueryToMarkJoinSlot, + Map subqueryCorrespondingConject, + CascadesContext ctx, Optional conjunct, + boolean isProject, boolean singleSubquery) { ctx.setSubqueryExprIsAnalyzed(subquery, true); LogicalApply newApply = new LogicalApply( subquery.getCorrelateSlots(), - subquery, Optional.empty(), childPlan, subquery.getQueryPlan()); - List projects = new ArrayList<>(childPlan.getOutput()); + subquery, Optional.empty(), + subqueryToMarkJoinSlot.get(subquery), + mergeScalarSubConjectAndFilterConject( + subquery, subqueryCorrespondingConject, + conjunct, isProject, singleSubquery), + childPlan, subquery.getQueryPlan()); + List projects = new ArrayList<>(childPlan.getOutput()); if (subquery instanceof ScalarSubquery) { projects.add(subquery.getQueryPlan().getOutput().get(0)); } return new LogicalProject(projects, newApply); } + private boolean checkSingleScalarWithOr(SubqueryExpr subquery, + Optional conjunct) { + return subquery instanceof ScalarSubquery + && conjunct.isPresent() && conjunct.get() instanceof Or + && subquery.getCorrelateSlots().isEmpty(); + } + + /** + * For a single scalarSubQuery, when there is a disjunction, + * directly use all connection conditions as the join conjunct of scalarSubQuery. + * e.g. + * select * from t1 where k1 > scalarSub(sum(c1)) or k2 > 10; + * LogicalJoin(otherConjunct[k1 > sum(c1) or k2 > 10]) + * + * For other scalarSubQuery, you only need to use the connection as the join conjunct. + * e.g. + * select * from t1 where k1 > scalarSub(sum(c1)) or k2 in inSub(c2) or k2 > 10; + * LogicalFilter($c$1 or $c$2 or k2 > 10) + * LogicalJoin(otherConjunct[k2 = c2]) ---> inSub + * LogicalJoin(otherConjunct[k1 > sum(c1)]) ---> scalarSub + */ + private Optional mergeScalarSubConjectAndFilterConject( + SubqueryExpr subquery, + Map subqueryCorrespondingConject, + Optional conjunct, + boolean isProject, + boolean singleSubquery) { + if (singleSubquery && checkSingleScalarWithOr(subquery, conjunct)) { + return conjunct; + } else if (subqueryCorrespondingConject.containsKey(subquery) && !isProject) { + return Optional.of(subqueryCorrespondingConject.get(subquery)); + } + return Optional.empty(); + } + /** * The Subquery in the LogicalFilter will change to LogicalApply, so we must replace the origin Subquery. * LogicalFilter(predicate(contain subquery)) -> LogicalFilter(predicate(not contain subquery) @@ -133,32 +235,207 @@ private LogicalPlan addApply(SubqueryExpr subquery, Plan childPlan, CascadesCont * * after: * 1.filter(t1.a = b); - * 2.filter(True); - * 3.filter(True); + * 2.isMarkJoin ? filter(MarkJoinSlotReference) : filter(True); + * 3.isMarkJoin ? filter(MarkJoinSlotReference) : filter(True); */ - private static class ReplaceSubquery extends DefaultExpressionRewriter { - public Set replace(Set expressions) { - return expressions.stream().map(expr -> expr.accept(this, null)) + private static class ReplaceSubquery extends DefaultExpressionRewriter { + private final StatementContext statementContext; + private boolean isMarkJoin; + + private final boolean isProject; + + public ReplaceSubquery(StatementContext statementContext, + boolean isProject) { + this.statementContext = Objects.requireNonNull(statementContext, "statementContext can't be null"); + this.isProject = isProject; + } + + public Set replace(Set expressions, SubqueryContext subqueryContext) { + return expressions.stream().map(expr -> expr.accept(this, subqueryContext)) .collect(ImmutableSet.toImmutableSet()); } - public Expression replace(Expression expressions) { - return expressions.accept(this, null); + public Expression replace(Expression expressions, SubqueryContext subqueryContext) { + return expressions.accept(this, subqueryContext); + } + + @Override + public Expression visitExistsSubquery(Exists exists, SubqueryContext context) { + // The result set when NULL is specified in the subquery and still evaluates to TRUE by using EXISTS + // When the number of rows returned is empty, agg will return null, so if there is more agg, + // it will always consider the returned result to be true + MarkJoinSlotReference markJoinSlotReference; + if (exists.getQueryPlan().anyMatch(Aggregate.class::isInstance)) { + markJoinSlotReference = + new MarkJoinSlotReference(statementContext.generateColumnName(), true); + } else { + markJoinSlotReference = + new MarkJoinSlotReference(statementContext.generateColumnName()); + } + if (isMarkJoin) { + context.setSubqueryToMarkJoinSlot(exists, Optional.of(markJoinSlotReference)); + } + return isMarkJoin ? markJoinSlotReference : BooleanLiteral.TRUE; + } + + @Override + public Expression visitInSubquery(InSubquery in, SubqueryContext context) { + MarkJoinSlotReference markJoinSlotReference = + new MarkJoinSlotReference(statementContext.generateColumnName()); + if (isMarkJoin) { + context.setSubqueryToMarkJoinSlot(in, Optional.of(markJoinSlotReference)); + } + return isMarkJoin ? markJoinSlotReference : BooleanLiteral.TRUE; } @Override - public Expression visitExistsSubquery(Exists exists, Void context) { - return BooleanLiteral.TRUE; + public Expression visitScalarSubquery(ScalarSubquery scalar, SubqueryContext context) { + context.setSubqueryCorrespondingConject(scalar, scalar.getQueryPlan().getOutput().get(0)); + // When there is only one scalarSubQuery and CorrelateSlots is empty + // it will not be processed by MarkJoin, so it can be returned directly + if (context.onlySingleSubquery() && scalar.getCorrelateSlots().isEmpty()) { + return scalar.getQueryPlan().getOutput().get(0); + } + + MarkJoinSlotReference markJoinSlotReference = + new MarkJoinSlotReference(statementContext.generateColumnName()); + if (isMarkJoin) { + context.setSubqueryToMarkJoinSlot(scalar, Optional.of(markJoinSlotReference)); + } + return isMarkJoin ? markJoinSlotReference : scalar.getQueryPlan().getOutput().get(0); } @Override - public Expression visitInSubquery(InSubquery in, Void context) { - return BooleanLiteral.TRUE; + public Expression visitNot(Not not, SubqueryContext context) { + // Need to re-update scalarSubQuery unequal conditions into subqueryCorrespondingConject + if (not.child() instanceof BinaryOperator + && (((BinaryOperator) not.child()).left() instanceof ScalarSubquery + || ((BinaryOperator) not.child()).right() instanceof ScalarSubquery)) { + Expression newChild = replace(not.child(), context); + ScalarSubquery subquery = ((BinaryOperator) not.child()).left() instanceof ScalarSubquery + ? (ScalarSubquery) ((BinaryOperator) not.child()).left() + : (ScalarSubquery) ((BinaryOperator) not.child()).right(); + context.updateSubqueryCorrespondingConjunctInNot(subquery); + return context.getSubqueryToMarkJoinSlotValue(subquery).isPresent() ? newChild : new Not(newChild); + } + + return visit(not, context); } @Override - public Expression visitScalarSubquery(ScalarSubquery scalar, Void context) { - return scalar.getQueryPlan().getOutput().get(0); + public Expression visitBinaryOperator(BinaryOperator binaryOperator, SubqueryContext context) { + boolean atLeastOneChildIsScalarSubquery = + binaryOperator.left() instanceof ScalarSubquery || binaryOperator.right() instanceof ScalarSubquery; + boolean currentMarkJoin = ((binaryOperator.left().anyMatch(SubqueryExpr.class::isInstance) + || binaryOperator.right().anyMatch(SubqueryExpr.class::isInstance)) + && (binaryOperator instanceof Or)) || isMarkJoin; + isMarkJoin = currentMarkJoin; + Expression left = replace(binaryOperator.left(), context); + isMarkJoin = currentMarkJoin; + Expression right = replace(binaryOperator.right(), context); + + if (atLeastOneChildIsScalarSubquery) { + return context.replaceBinaryOperator(binaryOperator, left, right, isProject); + } + return binaryOperator.withChildren(left, right); + } + } + + /** + * subqueryToMarkJoinSlot: The markJoinSlot corresponding to each subquery. + * rule: + * For inSubquery and exists: it will be directly replaced by markSlotReference + * e.g. + * logicalFilter(predicate=exists) ---> logicalFilter(predicate=$c$1) + * For scalarSubquery: will replace the connected ComparisonPredicate with markSlotReference + * e.g. + * logicalFilter(predicate=k1 > scalarSubquery) ---> logicalFilter(predicate=$c$1) + * + * subqueryCorrespondingConject: Record the conject corresponding to the subquery. + * rule: + * + * + */ + private static class SubqueryContext { + private final Map> subqueryToMarkJoinSlot; + + private final Map subqueryCorrespondingConjunct; + + public SubqueryContext(Set subqueryExprs) { + this.subqueryToMarkJoinSlot = new LinkedHashMap<>(subqueryExprs.size()); + this.subqueryCorrespondingConjunct = new LinkedHashMap<>(subqueryExprs.size()); + subqueryExprs.forEach(subqueryExpr -> subqueryToMarkJoinSlot.put(subqueryExpr, Optional.empty())); + } + + public Map> getSubqueryToMarkJoinSlot() { + return subqueryToMarkJoinSlot; + } + + public Map getSubqueryCorrespondingConjunct() { + return subqueryCorrespondingConjunct; + } + + public Optional getSubqueryToMarkJoinSlotValue(SubqueryExpr subqueryExpr) { + return subqueryToMarkJoinSlot.get(subqueryExpr); + } + + public void setSubqueryToMarkJoinSlot(SubqueryExpr subquery, + Optional markJoinSlotReference) { + subqueryToMarkJoinSlot.put(subquery, markJoinSlotReference); + } + + public void setSubqueryCorrespondingConject(SubqueryExpr subquery, + Expression expression) { + subqueryCorrespondingConjunct.put(subquery, expression); + } + + public boolean onlySingleSubquery() { + return subqueryToMarkJoinSlot.size() == 1; + } + + public void updateSubqueryCorrespondingConjunctInNot(SubqueryExpr subquery) { + if (subqueryCorrespondingConjunct.containsKey(subquery)) { + subqueryCorrespondingConjunct.replace(subquery, + new Not(subqueryCorrespondingConjunct.get(subquery))); + } + } + + /** + * For scalarSubQuery and MarkJoin, it will be replaced by markSlotReference + * e.g. + * logicalFilter(predicate=k1 > scalarSub or exists) + * --> + * logicalFilter(predicate=$c$1 or $c$2) + * + * For non-MarkJoin scalarSubQuery, do implicit type conversion. + * e.g. + * logicalFilter(predicate=k1 > scalarSub(sum(k2))) + * --> + * logicalFilter(predicate=Cast(k1[#0] as BIGINT) = sum(k2)[#1]) + */ + public Expression replaceBinaryOperator(BinaryOperator binaryOperator, + Expression left, + Expression right, + boolean isProject) { + boolean leftIsScalar = binaryOperator.left() instanceof ScalarSubquery; + ScalarSubquery subquery = leftIsScalar + ? (ScalarSubquery) binaryOperator.left() : (ScalarSubquery) binaryOperator.right(); + + // Perform implicit type conversion on the connection condition of scalarSubQuery, + // and record the result in subqueryCorrespondingConjunct + Expression newLeft = leftIsScalar && subqueryToMarkJoinSlot.get(subquery).isPresent() + ? ((ScalarSubquery) binaryOperator.left()).getQueryPlan().getOutput().get(0) : left; + Expression newRight = !leftIsScalar && subqueryToMarkJoinSlot.get(subquery).isPresent() + ? ((ScalarSubquery) binaryOperator.right()).getQueryPlan().getOutput().get(0) : right; + Expression newBinary = TypeCoercionUtils.processComparisonPredicate( + (ComparisonPredicate) binaryOperator.withChildren(newLeft, newRight), newLeft, newRight); + subqueryCorrespondingConjunct.put(subquery, + (isProject ? (leftIsScalar ? newLeft : newRight) : newBinary)); + + if (subqueryToMarkJoinSlot.get(subquery).isPresent()) { + return subqueryToMarkJoinSlot.get(subquery).get(); + } + return newBinary; } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscom.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscom.java index a16b7b91c106ac..a6f3a5ab2682e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscom.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscom.java @@ -52,6 +52,7 @@ public Rule build() { return innerLogicalJoin(innerLogicalJoin(), group()) .when(topJoin -> checkReorder(topJoin, topJoin.left())) .whenNot(join -> join.hasJoinHint() || join.left().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().isMarkJoin()) .then(topJoin -> { LogicalJoin bottomJoin = topJoin.left(); GroupPlan a = bottomJoin.left(); @@ -93,7 +94,8 @@ public Rule build() { public static boolean checkReorder(LogicalJoin topJoin, LogicalJoin bottomJoin) { return !bottomJoin.getJoinReorderContext().hasCommuteZigZag() - && !topJoin.getJoinReorderContext().hasLAsscom(); + && !topJoin.getJoinReorderContext().hasLAsscom() + && (!bottomJoin.isMarkJoin() && !topJoin.isMarkJoin()); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java index 3581bb81c03897..c23dfd5572c8a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLAsscomProject.java @@ -58,6 +58,7 @@ public Rule build() { return innerLogicalJoin(logicalProject(innerLogicalJoin()), group()) .when(topJoin -> InnerJoinLAsscom.checkReorder(topJoin, topJoin.left().child())) .whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().child().isMarkJoin()) .when(join -> JoinReorderUtils.isAllSlotProject(join.left())) .then(topJoin -> { /* ********** init ********** */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociate.java index 2f894953652e9d..103c6d67631458 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinLeftAssociate.java @@ -53,6 +53,7 @@ public Rule build() { return innerLogicalJoin(group(), innerLogicalJoin()) .when(InnerJoinLeftAssociate::checkReorder) .whenNot(join -> join.hasJoinHint() || join.right().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.right().isMarkJoin()) .then(topJoin -> { LogicalJoin bottomJoin = topJoin.right(); GroupPlan a = topJoin.left(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociate.java index b6ea7d2ad7a4b5..96d94c3d90e8fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/InnerJoinRightAssociate.java @@ -52,6 +52,7 @@ public Rule build() { return innerLogicalJoin(innerLogicalJoin(), group()) .when(InnerJoinRightAssociate::checkReorder) .whenNot(join -> join.hasJoinHint() || join.left().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().isMarkJoin()) .then(topJoin -> { LogicalJoin bottomJoin = topJoin.left(); GroupPlan a = bottomJoin.left(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java index 7b5b1c8a45d07a..d9110fb7d775e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinCommute.java @@ -46,6 +46,7 @@ public Rule build() { return logicalJoin() .when(join -> check(swapType, join)) .whenNot(LogicalJoin::hasJoinHint) + .whenNot(LogicalJoin::isMarkJoin) .then(join -> { LogicalJoin newJoin = new LogicalJoin<>( join.getJoinType().swap(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java index e95bd2310a8619..9ca232be9df26b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/JoinExchange.java @@ -55,6 +55,7 @@ public Rule build() { return innerLogicalJoin(innerLogicalJoin(), innerLogicalJoin()) .when(JoinExchange::checkReorder) .whenNot(join -> join.hasJoinHint() || join.left().hasJoinHint() || join.right().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().isMarkJoin() || join.right().isMarkJoin()) .then(topJoin -> { LogicalJoin leftJoin = topJoin.left(); LogicalJoin rightJoin = topJoin.right(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java index acb35495621740..370e92220e95c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java @@ -57,6 +57,7 @@ public Rule build() { .when(join -> VALID_TYPE_PAIR_SET.contains(Pair.of(join.left().getJoinType(), join.getJoinType()))) .when(topJoin -> OuterJoinLAsscom.checkReorder(topJoin, topJoin.left())) .when(topJoin -> checkCondition(topJoin, topJoin.left().left().getOutputSet())) + .whenNot(join -> join.isMarkJoin() || join.left().isMarkJoin()) .then(topJoin -> { LogicalJoin bottomJoin = topJoin.left(); GroupPlan a = bottomJoin.left(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java index b7d62584bc1e9b..06421c9c56a452 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java @@ -56,6 +56,7 @@ public Rule build() { Pair.of(join.left().child().getJoinType(), join.getJoinType()))) .when(topJoin -> OuterJoinLAsscom.checkReorder(topJoin, topJoin.left().child())) .whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().child().isMarkJoin()) .when(join -> OuterJoinAssoc.checkCondition(join, join.left().child().left().getOutputSet())) .when(join -> JoinReorderUtils.isAllSlotProject(join.left())) .then(topJoin -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscom.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscom.java index acf3a7e30cdc12..f2671958a05f3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscom.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscom.java @@ -64,6 +64,7 @@ public Rule build() { .when(topJoin -> checkReorder(topJoin, topJoin.left())) .whenNot(join -> join.hasJoinHint() || join.left().hasJoinHint()) .when(topJoin -> checkCondition(topJoin, topJoin.left().right().getOutputExprIdSet())) + .whenNot(LogicalJoin::isMarkJoin) .then(topJoin -> { LogicalJoin bottomJoin = topJoin.left(); GroupPlan a = bottomJoin.left(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java index 8756df12ea5231..266194718904d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinLAsscomProject.java @@ -59,6 +59,7 @@ public Rule build() { Pair.of(join.left().child().getJoinType(), join.getJoinType()))) .when(topJoin -> OuterJoinLAsscom.checkReorder(topJoin, topJoin.left().child())) .whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().child().isMarkJoin()) .when(join -> JoinReorderUtils.isAllSlotProject(join.left())) .then(topJoin -> { /* ********** init ********** */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java index cbefea40f6f1a0..0b57ed1b702a60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTranspose.java @@ -61,6 +61,7 @@ public Rule build() { .whenNot(topJoin -> topJoin.left().getJoinType().isSemiOrAntiJoin()) .when(this::conditionChecker) .whenNot(topJoin -> topJoin.hasJoinHint() || topJoin.left().hasJoinHint()) + .whenNot(LogicalJoin::isMarkJoin) .then(topSemiJoin -> { LogicalJoin bottomJoin = topSemiJoin.left(); GroupPlan a = bottomJoin.left(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java index 374fe5758b4e17..afa88aad98398e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinLogicalJoinTransposeProject.java @@ -67,6 +67,7 @@ public Rule build() { || topJoin.left().child().getJoinType().isRightOuterJoin()))) .whenNot(topJoin -> topJoin.left().child().getJoinType().isSemiOrAntiJoin()) .whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().child().isMarkJoin()) .when(join -> JoinReorderUtils.isAllSlotProject(join.left())) .then(topSemiJoin -> { LogicalProject> project = topSemiJoin.left(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTranspose.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTranspose.java index f2e5c50aeeac9f..6977dd9d62123c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTranspose.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTranspose.java @@ -63,6 +63,7 @@ public Rule build() { return logicalJoin(logicalJoin(), group()) .when(this::typeChecker) .whenNot(join -> join.hasJoinHint() || join.left().hasJoinHint()) + .whenNot(LogicalJoin::isMarkJoin) .then(topJoin -> { LogicalJoin bottomJoin = topJoin.left(); GroupPlan a = bottomJoin.left(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java index 85ac3cc27b8fc1..73af2fc191e531 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/SemiJoinSemiJoinTransposeProject.java @@ -54,6 +54,7 @@ public Rule build() { .when(this::typeChecker) .when(topSemi -> InnerJoinLAsscom.checkReorder(topSemi, topSemi.left().child())) .whenNot(join -> join.hasJoinHint() || join.left().child().hasJoinHint()) + .whenNot(join -> join.isMarkJoin() || join.left().child().isMarkJoin()) .when(join -> JoinReorderUtils.isAllSlotProject(join.left())) .then(topSemi -> { LogicalJoin bottomSemi = topSemi.left().child(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java index 50b83de0951a8d..a8d1a831cb5b1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/ExpressionRewrite.java @@ -204,7 +204,8 @@ public Rule build() { return join; } return new LogicalJoin<>(join.getJoinType(), rewriteHashJoinConjuncts, - rewriteOtherJoinConjuncts, join.getHint(), join.left(), join.right()); + rewriteOtherJoinConjuncts, join.getHint(), join.getMarkJoinSlotReference(), + join.left(), join.right()); }).toRule(RuleType.REWRITE_JOIN_EXPRESSION); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java index 8657bb48013048..0c175f0d63b7de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToHashJoin.java @@ -35,6 +35,7 @@ public Rule build() { join.getHashJoinConjuncts(), join.getOtherJoinConjuncts(), join.getHint(), + join.getMarkJoinSlotReference(), join.getLogicalProperties(), join.left(), join.right()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java index 7de8da2ec47f21..af02f6a2ed1202 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJoinToNestedLoopJoin.java @@ -34,6 +34,7 @@ public Rule build() { join.getJoinType(), join.getHashJoinConjuncts(), join.getOtherJoinConjuncts(), + join.getMarkJoinSlotReference(), join.getLogicalProperties(), join.left(), join.right()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExistsApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExistsApplyToJoin.java index a7447f1b357fa7..26a3f7de804b7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExistsApplyToJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExistsApplyToJoin.java @@ -91,20 +91,31 @@ public Rule build() { private Plan correlatedToJoin(LogicalApply apply) { Optional correlationFilter = apply.getCorrelationFilter(); + Expression predicate = null; + if (correlationFilter.isPresent() && apply.getSubCorrespondingConject().isPresent()) { + predicate = ExpressionUtils.and(correlationFilter.get(), + (Expression) apply.getSubCorrespondingConject().get()); + } else if (apply.getSubCorrespondingConject().isPresent()) { + predicate = (Expression) apply.getSubCorrespondingConject().get(); + } else if (correlationFilter.isPresent()) { + predicate = correlationFilter.get(); + } if (((Exists) apply.getSubqueryExpr()).isNot()) { return new LogicalJoin<>(JoinType.LEFT_ANTI_JOIN, ExpressionUtils.EMPTY_CONDITION, - correlationFilter - .map(ExpressionUtils::extractConjunction) - .orElse(ExpressionUtils.EMPTY_CONDITION), + predicate != null + ? ExpressionUtils.extractConjunction(predicate) + : ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, + apply.getMarkJoinSlotReference(), (LogicalPlan) apply.left(), (LogicalPlan) apply.right()); } else { return new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, ExpressionUtils.EMPTY_CONDITION, - correlationFilter - .map(ExpressionUtils::extractConjunction) - .orElse(ExpressionUtils.EMPTY_CONDITION), + predicate != null + ? ExpressionUtils.extractConjunction(predicate) + : ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, + apply.getMarkJoinSlotReference(), (LogicalPlan) apply.left(), (LogicalPlan) apply.right()); } } @@ -122,7 +133,10 @@ private Plan unCorrelatedNotExist(LogicalApply unapply) { Alias alias = new Alias(new Count(), "count(*)"); LogicalAggregate newAgg = new LogicalAggregate<>(new ArrayList<>(), ImmutableList.of(alias), newLimit); - LogicalJoin newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, + LogicalJoin newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, ExpressionUtils.EMPTY_CONDITION, + unapply.getSubCorrespondingConject().isPresent() + ? ExpressionUtils.extractConjunction((Expression) unapply.getSubCorrespondingConject().get()) + : ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, unapply.getMarkJoinSlotReference(), (LogicalPlan) unapply.left(), newAgg); return new LogicalFilter<>(ImmutableSet.of(new EqualTo(newAgg.getOutput().get(0), new IntegerLiteral(0))), newJoin); @@ -130,6 +144,10 @@ private Plan unCorrelatedNotExist(LogicalApply unapply) { private Plan unCorrelatedExist(LogicalApply unapply) { LogicalLimit newLimit = new LogicalLimit<>(1, 0, LimitPhase.ORIGIN, (LogicalPlan) unapply.right()); - return new LogicalJoin<>(JoinType.CROSS_JOIN, (LogicalPlan) unapply.left(), newLimit); + return new LogicalJoin<>(JoinType.CROSS_JOIN, ExpressionUtils.EMPTY_CONDITION, + unapply.getSubCorrespondingConject().isPresent() + ? ExpressionUtils.extractConjunction((Expression) unapply.getSubCorrespondingConject().get()) + : ExpressionUtils.EMPTY_CONDITION, + JoinHint.NONE, unapply.getMarkJoinSlotReference(), (LogicalPlan) unapply.left(), newLimit); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java index 6d01541d3e9a99..2c09eb8add35ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ExtractFilterFromCrossJoin.java @@ -42,7 +42,7 @@ public Rule build() { .then(join -> { LogicalJoin newJoin = new LogicalJoin<>(JoinType.CROSS_JOIN, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, join.getHint(), - join.left(), join.right()); + join.getMarkJoinSlotReference(), join.left(), join.right()); Set predicates = Stream.concat(join.getHashJoinConjuncts().stream(), join.getOtherJoinConjuncts().stream()) .collect(ImmutableSet.toImmutableSet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java index 9700048e5b6f20..d482969d3eff0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoin.java @@ -74,6 +74,7 @@ public Rule build() { combinedHashJoinConjuncts, remainedNonHashJoinConjuncts, join.getHint(), + join.getMarkJoinSlotReference(), join.left(), join.right()); }).toRule(RuleType.FIND_HASH_CONDITION_FOR_JOIN); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InApplyToJoin.java index a5cc4c9d14e778..d08125406aefd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InApplyToJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InApplyToJoin.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.types.BitmapType; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; import com.google.common.collect.Lists; @@ -46,14 +47,19 @@ public class InApplyToJoin extends OneRewriteRuleFactory { public Rule build() { return logicalApply().when(LogicalApply::isIn).then(apply -> { Expression predicate; + Expression left = ((InSubquery) apply.getSubqueryExpr()).getCompareExpr(); + Expression right = apply.right().getOutput().get(0); if (apply.isCorrelated()) { predicate = ExpressionUtils.and( - new EqualTo(((InSubquery) apply.getSubqueryExpr()).getCompareExpr(), - apply.right().getOutput().get(0)), + TypeCoercionUtils.processComparisonPredicate( + new EqualTo(left, right), left, right), apply.getCorrelationFilter().get()); } else { - predicate = new EqualTo(((InSubquery) apply.getSubqueryExpr()).getCompareExpr(), - apply.right().getOutput().get(0)); + predicate = TypeCoercionUtils.processComparisonPredicate(new EqualTo(left, right), left, right); + } + + if (apply.getSubCorrespondingConject().isPresent()) { + predicate = ExpressionUtils.and(predicate, apply.getSubCorrespondingConject().get()); } //TODO nereids should support bitmap runtime filter in future @@ -67,12 +73,12 @@ public Rule build() { predicate.nullable() ? JoinType.NULL_AWARE_LEFT_ANTI_JOIN : JoinType.LEFT_ANTI_JOIN, Lists.newArrayList(), conjuncts, - JoinHint.NONE, + JoinHint.NONE, apply.getMarkJoinSlotReference(), apply.left(), apply.right()); } else { return new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, Lists.newArrayList(), conjuncts, - JoinHint.NONE, + JoinHint.NONE, apply.getMarkJoinSlotReference(), apply.left(), apply.right()); } }).toRule(RuleType.IN_APPLY_TO_JOIN); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java index 403008a9d06aa7..e1cf58c789cbf8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpCorrelatedFilterUnderApplyAggregateProject.java @@ -79,7 +79,8 @@ public Rule build() { LogicalFilter newFilter = new LogicalFilter<>(filter.getConjuncts(), newProject); LogicalAggregate newAgg = agg.withChildren(ImmutableList.of(newFilter)); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), - apply.getCorrelationFilter(), apply.left(), newAgg); + apply.getCorrelationFilter(), apply.getMarkJoinSlotReference(), + apply.getSubCorrespondingConject(), apply.left(), newAgg); }).toRule(RuleType.PULL_UP_CORRELATED_FILTER_UNDER_APPLY_AGGREGATE_PROJECT); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java index 7f993f17988d0a..a6bb7ce924dc18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PullUpProjectUnderApply.java @@ -57,7 +57,8 @@ public Rule build() { .then(apply -> { LogicalProject project = apply.right(); LogicalApply newCorrelate = new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), - apply.getCorrelationFilter(), apply.left(), project.child()); + apply.getCorrelationFilter(), apply.getMarkJoinSlotReference(), + apply.getSubCorrespondingConject(), apply.left(), project.child()); List newProjects = new ArrayList<>(); newProjects.addAll(apply.left().getOutput()); if (apply.getSubqueryExpr() instanceof ScalarSubquery) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java index b2d192c1635c09..2e7b06173e16bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushFilterInsideJoin.java @@ -40,6 +40,7 @@ public class PushFilterInsideJoin extends OneRewriteRuleFactory { @Override public Rule build() { return logicalFilter(logicalJoin()) + .whenNot(filter -> filter.child().isMarkJoin()) // TODO: current just handle cross/inner join. .when(filter -> filter.child().getJoinType().isCrossJoin() || filter.child().getJoinType().isInnerJoin()) @@ -48,7 +49,8 @@ public Rule build() { LogicalJoin join = filter.child(); otherConditions.addAll(join.getOtherJoinConjuncts()); return new LogicalJoin<>(join.getJoinType(), join.getHashJoinConjuncts(), - otherConditions, join.getHint(), join.left(), join.right()); + otherConditions, join.getHint(), join.getMarkJoinSlotReference(), + join.left(), join.right()); }).toRule(RuleType.PUSH_FILTER_INSIDE_JOIN); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java index 64ff0cd514e605..34657f655dde04 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoin.java @@ -139,6 +139,7 @@ public Rule build() { join.getHashJoinConjuncts(), joinConditions, join.getHint(), + join.getMarkJoinSlotReference(), PlanUtils.filterOrSelf(leftPredicates, join.left()), PlanUtils.filterOrSelf(rightPredicates, join.right()))); }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_JOIN); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherCondition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherCondition.java index 46f561372abb4a..1f06bdf2f8caca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherCondition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherCondition.java @@ -90,7 +90,7 @@ && allCoveredBy(otherConjunct, join.right().getOutputSet())) { Plan right = PlanUtils.filterOrSelf(rightConjuncts, join.right()); return new LogicalJoin<>(join.getJoinType(), join.getHashJoinConjuncts(), - remainingOther, join.getHint(), left, right); + remainingOther, join.getHint(), join.getMarkJoinSlotReference(), left, right); }).toRule(RuleType.PUSHDOWN_JOIN_OTHER_CONDITION); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java index ede6586a3a4e7c..77d57e6accf2ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinHint.JoinHintType; import org.apache.doris.nereids.trees.plans.JoinType; @@ -45,6 +46,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -80,12 +82,12 @@ public Rule build() { } LogicalFilter filter = ctx.root; - Map planToHintType = Maps.newHashMap(); - Plan plan = joinToMultiJoin(filter, planToHintType); + Map planToJoinMembers = Maps.newHashMap(); + Plan plan = joinToMultiJoin(filter, planToJoinMembers); Preconditions.checkState(plan instanceof MultiJoin); MultiJoin multiJoin = (MultiJoin) plan; ctx.statementContext.setMaxNArayInnerJoin(multiJoin.children().size()); - Plan after = multiJoinToJoin(multiJoin, planToHintType); + Plan after = multiJoinToJoin(multiJoin, planToJoinMembers); return after; }).toRule(RuleType.REORDER_JOIN); } @@ -95,7 +97,7 @@ public Rule build() { * {@link LogicalJoin} or {@link LogicalFilter}--{@link LogicalJoin} * --> {@link MultiJoin} */ - public Plan joinToMultiJoin(Plan plan, Map planToHintType) { + public Plan joinToMultiJoin(Plan plan, Map planToJoinMembers) { // subtree can't specify the end of Pattern. so end can be GroupPlan or Filter if (nonJoinAndNonFilter(plan) || (plan instanceof LogicalFilter && nonJoinAndNonFilter(plan.child(0)))) { @@ -125,10 +127,12 @@ public Plan joinToMultiJoin(Plan plan, Map planToHintType) { } // recursively convert children. - planToHintType.put(join.left(), join.getLeftHint()); - Plan left = joinToMultiJoin(join.left(), planToHintType); - planToHintType.put(join.right(), join.getRightHint()); - Plan right = joinToMultiJoin(join.right(), planToHintType); + planToJoinMembers.put(join.left(), + new JoinHintTypeAndMarkJoinSlot(join.getLeftHint(), join.getLeftMarkJoinSlotReference())); + Plan left = joinToMultiJoin(join.left(), planToJoinMembers); + planToJoinMembers.put(join.right(), + new JoinHintTypeAndMarkJoinSlot(join.getRightHint(), join.getMarkJoinSlotReference())); + Plan right = joinToMultiJoin(join.right(), planToJoinMembers); boolean changeLeft = join.getJoinType().isRightJoin() || join.getJoinType().isFullOuterJoin(); @@ -211,7 +215,7 @@ public Plan joinToMultiJoin(Plan plan, Map planToHintType) { * A B C D F ──► A B C │ D F ──► MJ(FOJ MJ(A,B,C) MJ(D,F)) * */ - public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToHintType) { + public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToJoinMembers) { if (multiJoin.arity() == 1) { return PlanUtils.filterOrSelf(ImmutableSet.copyOf(multiJoin.getJoinFilter()), multiJoin.child(0)); } @@ -221,7 +225,7 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToH for (Plan child : multiJoin.children()) { if (child instanceof MultiJoin) { MultiJoin childMultiJoin = (MultiJoin) child; - builder.add(multiJoinToJoin(childMultiJoin, planToHintType)); + builder.add(multiJoinToJoin(childMultiJoin, planToJoinMembers)); } else { builder.add(child); } @@ -239,6 +243,7 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToH Map> split = multiJoin.getJoinFilter().stream() .collect(Collectors.partitioningBy(expr -> Utils.isIntersecting(rightOutputExprIdSet, expr.getInputSlotExprIds()) + || expr.anyMatch(MarkJoinSlotReference.class::isInstance) )); remainingFilter = split.get(true); List pushedFilter = split.get(false); @@ -246,7 +251,7 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToH multiJoinHandleChildren.children().subList(0, multiJoinHandleChildren.arity() - 1), pushedFilter, JoinType.INNER_JOIN, - ExpressionUtils.EMPTY_CONDITION), planToHintType); + ExpressionUtils.EMPTY_CONDITION), planToJoinMembers); } else if (multiJoinHandleChildren.getJoinType().isRightJoin()) { left = multiJoinHandleChildren.child(0); Set leftOutputExprIdSet = left.getOutputExprIdSet(); @@ -260,13 +265,13 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToH multiJoinHandleChildren.children().subList(1, multiJoinHandleChildren.arity()), pushedFilter, JoinType.INNER_JOIN, - ExpressionUtils.EMPTY_CONDITION), planToHintType); + ExpressionUtils.EMPTY_CONDITION), planToJoinMembers); } else { remainingFilter = multiJoin.getJoinFilter(); Preconditions.checkState(multiJoinHandleChildren.arity() == 2); List children = multiJoinHandleChildren.children().stream().map(child -> { if (child instanceof MultiJoin) { - return multiJoinToJoin((MultiJoin) child, planToHintType); + return multiJoinToJoin((MultiJoin) child, planToJoinMembers); } else { return child; } @@ -278,7 +283,8 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToH return PlanUtils.filterOrSelf(ImmutableSet.copyOf(remainingFilter), new LogicalJoin<>( multiJoinHandleChildren.getJoinType(), ExpressionUtils.EMPTY_CONDITION, multiJoinHandleChildren.getNotInnerJoinConditions(), - JoinHint.fromRightPlanHintType(planToHintType.getOrDefault(right, JoinHintType.NONE)), + JoinHint.fromRightPlanHintType(getJoinHintType(planToJoinMembers, right)), + getMarkJoinSlotReference(planToJoinMembers, right), left, right)); } @@ -291,7 +297,7 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planToH while (usedPlansIndex.size() != multiJoinHandleChildren.children().size()) { LogicalJoin join = findInnerJoin(left, multiJoinHandleChildren.children(), - joinFilter, usedPlansIndex, planToHintType); + joinFilter, usedPlansIndex, planToJoinMembers); join.getHashJoinConjuncts().forEach(joinFilter::remove); join.getOtherJoinConjuncts().forEach(joinFilter::remove); @@ -321,7 +327,8 @@ private static boolean canCombine(Plan input, boolean changeChildren) { * @return InnerJoin or CrossJoin{left, last of [candidates]} */ private LogicalJoin findInnerJoin(Plan left, List candidates, - Set joinFilter, Set usedPlansIndex, Map planToHintType) { + Set joinFilter, Set usedPlansIndex, + Map planToJoinMembers) { List otherJoinConditions = Lists.newArrayList(); Set leftOutputExprIdSet = left.getOutputExprIdSet(); for (int i = 0; i < candidates.size(); i++) { @@ -350,7 +357,8 @@ private static boolean canCombine(Plan input, boolean changeChildren) { usedPlansIndex.add(i); return new LogicalJoin<>(JoinType.INNER_JOIN, hashJoinConditions, otherJoinConditions, - JoinHint.fromRightPlanHintType(planToHintType.getOrDefault(candidate, JoinHintType.NONE)), + JoinHint.fromRightPlanHintType(getJoinHintType(planToJoinMembers, candidate)), + getMarkJoinSlotReference(planToJoinMembers, candidate), left, candidate); } } @@ -365,7 +373,8 @@ private static boolean canCombine(Plan input, boolean changeChildren) { return new LogicalJoin<>(JoinType.CROSS_JOIN, ExpressionUtils.EMPTY_CONDITION, otherJoinConditions, - JoinHint.fromRightPlanHintType(planToHintType.getOrDefault(right, JoinHintType.NONE)), + JoinHint.fromRightPlanHintType(getJoinHintType(planToJoinMembers, right)), + getMarkJoinSlotReference(planToJoinMembers, right), left, right); } @@ -375,4 +384,33 @@ private static boolean canCombine(Plan input, boolean changeChildren) { private boolean nonJoinAndNonFilter(Plan plan) { return !(plan instanceof LogicalJoin) && !(plan instanceof LogicalFilter); } + + private JoinHintType getJoinHintType(Map planToJoinMembers, Plan plan) { + return planToJoinMembers.get(plan) == null ? JoinHintType.NONE : planToJoinMembers.get(plan).getJoinHintType(); + } + + private Optional getMarkJoinSlotReference( + Map planToJoinMembers, Plan plan) { + return planToJoinMembers.get(plan) == null + ? Optional.empty() : planToJoinMembers.get(plan).getMarkJoinSlotReference(); + } + + private static class JoinHintTypeAndMarkJoinSlot { + private JoinHintType joinHintType; + private Optional markJoinSlotReference; + + public JoinHintTypeAndMarkJoinSlot( + JoinHintType joinHintType, Optional markJoinSlotReference) { + this.joinHintType = joinHintType; + this.markJoinSlotReference = markJoinSlotReference; + } + + public JoinHintType getJoinHintType() { + return joinHintType == null ? JoinHintType.NONE : joinHintType; + } + + public Optional getMarkJoinSlotReference() { + return markJoinSlotReference; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ScalarApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ScalarApplyToJoin.java index 97585cf24c9981..97a131582558b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ScalarApplyToJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ScalarApplyToJoin.java @@ -60,6 +60,12 @@ private Plan unCorrelatedToJoin(LogicalApply apply) { AssertNumRowsElement.Assertion.EQ), (LogicalPlan) apply.right()); return new LogicalJoin<>(JoinType.CROSS_JOIN, + ExpressionUtils.EMPTY_CONDITION, + apply.getSubCorrespondingConject().isPresent() + ? ExpressionUtils.extractConjunction((Expression) apply.getSubCorrespondingConject().get()) + : ExpressionUtils.EMPTY_CONDITION, + JoinHint.NONE, + apply.getMarkJoinSlotReference(), (LogicalPlan) apply.left(), assertNumRows); } @@ -73,14 +79,20 @@ private Plan correlatedToJoin(LogicalApply apply) { throw new AnalysisException( "scalar subquery's correlatedPredicates's operator must be EQ"); }); + } else { + throw new AnalysisException("correlationFilter can't be null in correlatedToJoin"); } - return new LogicalJoin<>(JoinType.LEFT_OUTER_JOIN, + return new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, ExpressionUtils.EMPTY_CONDITION, - correlationFilter - .map(ExpressionUtils::extractConjunction) - .orElse(ExpressionUtils.EMPTY_CONDITION), + ExpressionUtils.extractConjunction( + apply.getSubCorrespondingConject().isPresent() + ? ExpressionUtils.and( + (Expression) apply.getSubCorrespondingConject().get(), + correlationFilter.get()) + : correlationFilter.get()), JoinHint.NONE, + apply.getMarkJoinSlotReference(), (LogicalPlan) apply.left(), (LogicalPlan) apply.right()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java index 2b8543171149e5..ab05f239216841 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyAggregateFilter.java @@ -88,6 +88,8 @@ public Rule build() { return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), + apply.getMarkJoinSlotReference(), + apply.getSubCorrespondingConject(), apply.left(), newAgg); }).toRule(RuleType.UN_CORRELATED_APPLY_AGGREGATE_FILTER); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java index 1ffbcf22147771..95959a62059ede 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyFilter.java @@ -68,7 +68,8 @@ public Rule build() { Plan child = PlanUtils.filterOrSelf(ImmutableSet.copyOf(unCorrelatedPredicate), filter.child()); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), - ExpressionUtils.optionalAnd(correlatedPredicate), + ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(), + apply.getSubCorrespondingConject(), apply.left(), child); }).toRule(RuleType.UN_CORRELATED_APPLY_FILTER); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java index 23d9ec025af68d..0028c53bc7ec9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/UnCorrelatedApplyProjectFilter.java @@ -89,7 +89,8 @@ public Rule build() { .forEach(projects::add); LogicalProject newProject = new LogicalProject(projects, child); return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), - ExpressionUtils.optionalAnd(correlatedPredicate), + ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(), + apply.getSubCorrespondingConject(), apply.left(), newProject); }).toRule(RuleType.UN_CORRELATED_APPLY_PROJECT_FILTER); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java index 6722e7ea691408..59a948cdc51c28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/ExpressionEstimation.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IntegralDivide; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.Subtract; @@ -293,4 +294,10 @@ public ColumnStatistic visitTimestampArithmetic(TimestampArithmetic arithmetic, builder.setSelectivity(1.0); return builder.build(); } + + @Override + public ColumnStatistic visitMarkJoinReference( + MarkJoinSlotReference markJoinSlotReference, StatsDeriveResult context) { + return ColumnStatistic.DEFAULT; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AssertNumRowsElement.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AssertNumRowsElement.java index 44d15a979ef760..4a80d73e5da210 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AssertNumRowsElement.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/AssertNumRowsElement.java @@ -75,9 +75,9 @@ public AssertNumRowsElement withChildren(List children) { @Override public String toString() { return Utils.toSqlString("AssertNumRowsElement", - "desiredNumOfRows: ", + "desiredNumOfRows", Long.toString(desiredNumOfRows), - "assertion: ", assertion); + "assertion", assertion); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java new file mode 100644 index 00000000000000..ff841bdeb2558c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/MarkJoinSlotReference.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BooleanType; + +/** + * A special type of column that will be generated to replace the subquery when unnesting the subquery of MarkJoin. + */ +public class MarkJoinSlotReference extends SlotReference implements SlotNotFromChildren { + final boolean existsHasAgg; + + public MarkJoinSlotReference(String name) { + super(name, BooleanType.INSTANCE, false); + this.existsHasAgg = false; + } + + public MarkJoinSlotReference(String name, boolean existsHasAgg) { + super(name, BooleanType.INSTANCE, false); + this.existsHasAgg = existsHasAgg; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitMarkJoinReference(this, context); + } + + @Override + public String toString() { + return super.toString() + "#" + existsHasAgg; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MarkJoinSlotReference that = (MarkJoinSlotReference) o; + return this.existsHasAgg == that.existsHasAgg && super.equals(that); + } + + public boolean isExistsHasAgg() { + return existsHasAgg; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotNotFromChildren.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotNotFromChildren.java new file mode 100644 index 00000000000000..038fe3f949b561 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotNotFromChildren.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +/** + * CheckAfterRewrite will check whether the slots used by the current node can be obtained in the child's output, + * but there are special slots that are actively generated by the current node, + * so such slots need to be skipped during the check. + */ +public interface SlotNotFromChildren { +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java index f742dc88da7923..eaae4553d2c1db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/VirtualSlotReference.java @@ -32,7 +32,7 @@ /** * it is not a real column exist in table. */ -public class VirtualSlotReference extends SlotReference { +public class VirtualSlotReference extends SlotReference implements SlotNotFromChildren { // arguments of GroupingScalarFunction private final List realExpressions; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index 977070fa861f48..9f1922396eaa18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -51,6 +51,7 @@ import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.LessThanEqual; import org.apache.doris.nereids.trees.expressions.ListQuery; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Mod; import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -199,6 +200,10 @@ public R visitSlotReference(SlotReference slotReference, C context) { return visitSlot(slotReference, context); } + public R visitMarkJoinReference(MarkJoinSlotReference markJoinSlotReference, C context) { + return visitSlotReference(markJoinSlotReference, context); + } + public R visitLiteral(Literal literal, C context) { return visit(literal, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java index 6de43e7db35a4a..77bf6c9148ddf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Join.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.trees.plans.algebra; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinHint.JoinHintType; import org.apache.doris.nereids.trees.plans.JoinType; @@ -39,6 +40,8 @@ public interface Join { JoinHint getHint(); + boolean isMarkJoin(); + default boolean hasJoinHint() { return getHint() != JoinHint.NONE; } @@ -67,4 +70,8 @@ default JoinHintType getRightHint() { return JoinHintType.NONE; } } + + default Optional getLeftMarkJoinSlotReference() { + return Optional.empty(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java index 0452ae29dce2df..4c3a092244a55b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.InSubquery; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SubqueryExpr; @@ -51,6 +52,10 @@ public class LogicalApply correlationFilter; + // The slot replaced by the subquery in MarkJoin + private final Optional markJoinSlotReference; + + private final Optional subCorrespondingConject; /** * Constructor. @@ -59,18 +64,23 @@ public LogicalApply(Optional groupExpression, Optional logicalProperties, List correlationSlot, SubqueryExpr subqueryExpr, Optional correlationFilter, + Optional markJoinSlotReference, + Optional subCorrespondingConject, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { super(PlanType.LOGICAL_APPLY, groupExpression, logicalProperties, leftChild, rightChild); this.correlationSlot = correlationSlot == null ? ImmutableList.of() : ImmutableList.copyOf(correlationSlot); this.subqueryExpr = Objects.requireNonNull(subqueryExpr, "subquery can not be null"); this.correlationFilter = correlationFilter; + this.markJoinSlotReference = markJoinSlotReference; + this.subCorrespondingConject = subCorrespondingConject; } public LogicalApply(List correlationSlot, SubqueryExpr subqueryExpr, - Optional correlationFilter, + Optional correlationFilter, Optional markJoinSlotReference, + Optional subCorrespondingConject, LEFT_CHILD_TYPE input, RIGHT_CHILD_TYPE subquery) { - this(Optional.empty(), Optional.empty(), correlationSlot, subqueryExpr, correlationFilter, - input, subquery); + this(Optional.empty(), Optional.empty(), correlationSlot, subqueryExpr, + correlationFilter, markJoinSlotReference, subCorrespondingConject, input, subquery); } public List getCorrelationSlot() { @@ -105,6 +115,18 @@ public boolean alreadyExecutedEliminateFilter() { return correlationFilter.isPresent(); } + public boolean isMarkJoin() { + return markJoinSlotReference.isPresent(); + } + + public Optional getMarkJoinSlotReference() { + return markJoinSlotReference; + } + + public Optional getSubCorrespondingConject() { + return subCorrespondingConject; + } + @Override public List computeOutput() { return ImmutableList.builder() @@ -116,7 +138,11 @@ public List computeOutput() { public String toString() { return Utils.toSqlString("LogicalApply", "correlationSlot", correlationSlot, - "correlationFilter", correlationFilter); + "correlationFilter", correlationFilter, + "isMarkJoin", markJoinSlotReference.isPresent(), + "MarkJoinSlotReference", markJoinSlotReference.isPresent() ? markJoinSlotReference.get() : "empty", + "scalarSubCorrespondingSlot", + subCorrespondingConject.isPresent() ? subCorrespondingConject.get() : "empty"); } @Override @@ -130,13 +156,15 @@ public boolean equals(Object o) { LogicalApply that = (LogicalApply) o; return Objects.equals(correlationSlot, that.getCorrelationSlot()) && Objects.equals(subqueryExpr, that.getSubqueryExpr()) - && Objects.equals(correlationFilter, that.getCorrelationFilter()); + && Objects.equals(correlationFilter, that.getCorrelationFilter()) + && Objects.equals(markJoinSlotReference, that.getMarkJoinSlotReference()) + && Objects.equals(subCorrespondingConject, that.getSubCorrespondingConject()); } @Override public int hashCode() { return Objects.hash( - correlationSlot, subqueryExpr, correlationFilter); + correlationSlot, subqueryExpr, correlationFilter, markJoinSlotReference, subCorrespondingConject); } @Override @@ -161,18 +189,21 @@ public List getExpressions() { public LogicalBinary withChildren(List children) { Preconditions.checkArgument(children.size() == 2); return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter, + markJoinSlotReference, subCorrespondingConject, children.get(0), children.get(1)); } @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalApply<>(groupExpression, Optional.of(getLogicalProperties()), - correlationSlot, subqueryExpr, correlationFilter, left(), right()); + correlationSlot, subqueryExpr, correlationFilter, + markJoinSlotReference, subCorrespondingConject, left(), right()); } @Override public Plan withLogicalProperties(Optional logicalProperties) { return new LogicalApply<>(Optional.empty(), logicalProperties, - correlationSlot, subqueryExpr, correlationFilter, left(), right()); + correlationSlot, subqueryExpr, correlationFilter, + markJoinSlotReference, subCorrespondingConject, left(), right()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java index 9a4734213c37ff..6cb7dd60f7a91d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinType; @@ -55,31 +56,47 @@ public class LogicalJoin hashJoinConjuncts; private final JoinHint hint; + // When the predicate condition contains subqueries and disjunctions, the join will be marked as MarkJoin. + private final Optional markJoinSlotReference; + // Use for top-to-down join reorder private final JoinReorderContext joinReorderContext = new JoinReorderContext(); public LogicalJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { this(joinType, ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, - Optional.empty(), Optional.empty(), leftChild, rightChild); + Optional.empty(), Optional.empty(), Optional.empty(), leftChild, rightChild); } public LogicalJoin(JoinType joinType, List hashJoinConjuncts, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { this(joinType, hashJoinConjuncts, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), - Optional.empty(), leftChild, rightChild); + Optional.empty(), Optional.empty(), leftChild, rightChild); } public LogicalJoin(JoinType joinType, List hashJoinConjuncts, List otherJoinConjuncts, JoinHint hint, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - this(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, Optional.empty(), Optional.empty(), leftChild, - rightChild); + this(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, Optional.empty(), Optional.empty(), + Optional.empty(), leftChild, rightChild); + } + + public LogicalJoin( + JoinType joinType, + List hashJoinConjuncts, + List otherJoinConjuncts, + JoinHint hint, + Optional markJoinSlotReference, + LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { + this(joinType, hashJoinConjuncts, + otherJoinConjuncts, hint, markJoinSlotReference, + Optional.empty(), Optional.empty(), leftChild, rightChild); } /** * Just use in withXXX method. */ private LogicalJoin(JoinType joinType, List hashJoinConjuncts, List otherJoinConjuncts, - JoinHint hint, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, + JoinHint hint, Optional markJoinSlotReference, + LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, JoinReorderContext joinReorderContext) { super(PlanType.LOGICAL_JOIN, Optional.empty(), Optional.empty(), leftChild, rightChild); this.joinType = Objects.requireNonNull(joinType, "joinType can not be null"); @@ -87,6 +104,7 @@ private LogicalJoin(JoinType joinType, List hashJoinConjuncts, List< this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts); this.hint = Objects.requireNonNull(hint, "hint can not be null"); this.joinReorderContext.copyFrom(joinReorderContext); + this.markJoinSlotReference = markJoinSlotReference; } private LogicalJoin( @@ -94,6 +112,7 @@ private LogicalJoin( List hashJoinConjuncts, List otherJoinConjuncts, JoinHint hint, + Optional markJoinSlotReference, Optional groupExpression, Optional logicalProperties, LEFT_CHILD_TYPE leftChild, @@ -103,6 +122,7 @@ private LogicalJoin( this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts); this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts); this.hint = Objects.requireNonNull(hint, "hint can not be null"); + this.markJoinSlotReference = markJoinSlotReference; } public List getOtherJoinConjuncts() { @@ -131,6 +151,10 @@ public JoinHint getHint() { return hint; } + public boolean isMarkJoin() { + return markJoinSlotReference.isPresent(); + } + public JoinReorderContext getJoinReorderContext() { return joinReorderContext; } @@ -140,10 +164,16 @@ public List computeOutput() { return JoinUtils.getJoinOutput(joinType, left(), right()); } + @Override + public List computeNonUserVisibleOutput() { + return markJoinSlotReference.>map(ImmutableList::of).orElseGet(ImmutableList::of); + } + @Override public String toString() { List args = Lists.newArrayList( "type", joinType, + "markJoinSlotReference", markJoinSlotReference, "hashJoinConjuncts", hashJoinConjuncts, "otherJoinConjuncts", otherJoinConjuncts); if (hint != JoinHint.NONE) { @@ -168,12 +198,13 @@ public boolean equals(Object o) { return joinType == that.joinType && hint == that.hint && hashJoinConjuncts.equals(that.hashJoinConjuncts) - && otherJoinConjuncts.equals(that.otherJoinConjuncts); + && otherJoinConjuncts.equals(that.otherJoinConjuncts) + && Objects.equals(markJoinSlotReference, that.markJoinSlotReference); } @Override public int hashCode() { - return Objects.hash(joinType, hashJoinConjuncts, otherJoinConjuncts); + return Objects.hash(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference); } @Override @@ -189,6 +220,10 @@ public List getExpressions() { .build(); } + public Optional getMarkJoinSlotReference() { + return markJoinSlotReference; + } + @Override public LEFT_CHILD_TYPE left() { return (LEFT_CHILD_TYPE) child(0); @@ -202,13 +237,15 @@ public RIGHT_CHILD_TYPE right() { @Override public LogicalJoin withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, children.get(0), + return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + markJoinSlotReference, children.get(0), children.get(1), joinReorderContext); } @Override public LogicalJoin withGroupExpression(Optional groupExpression) { LogicalJoin newJoin = new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + markJoinSlotReference, groupExpression, Optional.of(getLogicalProperties()), left(), right()); newJoin.getJoinReorderContext().copyFrom(this.getJoinReorderContext()); return newJoin; @@ -217,42 +254,46 @@ public LogicalJoin withGroupExpression(Optional gro @Override public LogicalJoin withLogicalProperties(Optional logicalProperties) { LogicalJoin newJoin = new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + markJoinSlotReference, Optional.empty(), logicalProperties, left(), right()); newJoin.getJoinReorderContext().copyFrom(this.getJoinReorderContext()); return newJoin; } public LogicalJoin withHashJoinConjuncts(List hashJoinConjuncts) { - return new LogicalJoin<>(joinType, hashJoinConjuncts, this.otherJoinConjuncts, hint, left(), right(), - joinReorderContext); + return new LogicalJoin<>(joinType, hashJoinConjuncts, this.otherJoinConjuncts, hint, markJoinSlotReference, + left(), right(), joinReorderContext); } public LogicalJoin withJoinConjuncts( List hashJoinConjuncts, List otherJoinConjuncts) { return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, - hint, left(), right(), joinReorderContext); + hint, markJoinSlotReference, left(), right(), joinReorderContext); } public LogicalJoin withHashJoinConjunctsAndChildren( List hashJoinConjuncts, List children) { Preconditions.checkArgument(children.size() == 2); - return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, children.get(0), + return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + markJoinSlotReference, children.get(0), children.get(1), joinReorderContext); } public LogicalJoin withConjunctsChildren(List hashJoinConjuncts, List otherJoinConjuncts, Plan left, Plan right) { - return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, left, + return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, left, right, joinReorderContext); } public LogicalJoin withJoinType(JoinType joinType) { - return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, left(), right(), + return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + markJoinSlotReference, left(), right(), joinReorderContext); } public LogicalJoin withOtherJoinConjuncts(List otherJoinConjuncts) { - return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, left(), right(), + return new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + markJoinSlotReference, left(), right(), joinReorderContext); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java index a7c1484e4c9669..9454e2d879d199 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/UsingJoin.java @@ -20,6 +20,7 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinType; @@ -46,23 +47,28 @@ public class UsingJoin otherJoinConjuncts; private final ImmutableList hashJoinConjuncts; private final JoinHint hint; + private final Optional markJoinSlotReference; public UsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, List expressions, List hashJoinConjuncts, JoinHint hint) { this(joinType, leftChild, rightChild, expressions, - hashJoinConjuncts, Optional.empty(), Optional.empty(), hint); + hashJoinConjuncts, Optional.empty(), Optional.empty(), hint, Optional.empty()); } + /** + * Constructor. + */ public UsingJoin(JoinType joinType, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild, List expressions, List hashJoinConjuncts, Optional groupExpression, Optional logicalProperties, - JoinHint hint) { + JoinHint hint, Optional markJoinSlotReference) { super(PlanType.LOGICAL_USING_JOIN, groupExpression, logicalProperties, leftChild, rightChild); this.joinType = joinType; this.otherJoinConjuncts = ImmutableList.copyOf(expressions); this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts); this.hint = hint; + this.markJoinSlotReference = markJoinSlotReference; } @Override @@ -107,19 +113,19 @@ public List computeOutput() { @Override public Plan withGroupExpression(Optional groupExpression) { return new UsingJoin(joinType, child(0), child(1), otherJoinConjuncts, - hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint); + hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint, markJoinSlotReference); } @Override public Plan withLogicalProperties(Optional logicalProperties) { return new UsingJoin(joinType, child(0), child(1), otherJoinConjuncts, - hashJoinConjuncts, groupExpression, logicalProperties, hint); + hashJoinConjuncts, groupExpression, logicalProperties, hint, markJoinSlotReference); } @Override public Plan withChildren(List children) { return new UsingJoin(joinType, children.get(0), children.get(1), otherJoinConjuncts, - hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint); + hashJoinConjuncts, groupExpression, Optional.of(getLogicalProperties()), hint, markJoinSlotReference); } @Override @@ -151,6 +157,14 @@ public JoinHint getHint() { return hint; } + public boolean isMarkJoin() { + return markJoinSlotReference.isPresent(); + } + + public Optional getMarkJoinSlotReference() { + return markJoinSlotReference; + } + @Override public Optional getOnClauseCondition() { return ExpressionUtils.optionalAnd(hashJoinConjuncts, otherJoinConjuncts); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java index d1127a4647cd84..9055332774c980 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/AbstractPhysicalJoin.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -50,6 +51,7 @@ public abstract class AbstractPhysicalJoin< protected final List hashJoinConjuncts; protected final List otherJoinConjuncts; protected final JoinHint hint; + protected final Optional markJoinSlotReference; // use for translate only protected final List filterConjuncts = Lists.newArrayList(); @@ -64,6 +66,7 @@ public AbstractPhysicalJoin( List hashJoinConjuncts, List otherJoinConjuncts, JoinHint hint, + Optional markJoinSlotReference, Optional groupExpression, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { super(type, groupExpression, logicalProperties, leftChild, rightChild); @@ -71,6 +74,7 @@ public AbstractPhysicalJoin( this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts); this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts); this.hint = Objects.requireNonNull(hint, "hint can not be null"); + this.markJoinSlotReference = markJoinSlotReference; } /** @@ -82,6 +86,7 @@ public AbstractPhysicalJoin( List hashJoinConjuncts, List otherJoinConjuncts, JoinHint hint, + Optional markJoinSlotReference, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, @@ -93,6 +98,7 @@ public AbstractPhysicalJoin( this.hashJoinConjuncts = ImmutableList.copyOf(hashJoinConjuncts); this.otherJoinConjuncts = ImmutableList.copyOf(otherJoinConjuncts); this.hint = hint; + this.markJoinSlotReference = markJoinSlotReference; } public List getHashJoinConjuncts() { @@ -115,6 +121,10 @@ public List getOtherJoinConjuncts() { return otherJoinConjuncts; } + public boolean isMarkJoin() { + return markJoinSlotReference.isPresent(); + } + @Override public List getExpressions() { return new Builder() @@ -139,12 +149,13 @@ public boolean equals(Object o) { return joinType == that.joinType && hashJoinConjuncts.equals(that.hashJoinConjuncts) && otherJoinConjuncts.equals(that.otherJoinConjuncts) - && hint.equals(that.hint); + && hint.equals(that.hint) + && Objects.equals(markJoinSlotReference, that.markJoinSlotReference); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), joinType, hashJoinConjuncts, otherJoinConjuncts); + return Objects.hash(super.hashCode(), joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference); } /** @@ -165,6 +176,10 @@ public List getFilterConjuncts() { return filterConjuncts; } + public Optional getMarkJoinSlotReference() { + return markJoinSlotReference; + } + public void addFilterConjuncts(Collection conjuncts) { filterConjuncts.addAll(conjuncts); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java index ba66df69592382..59a416a7bd6b55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -48,11 +49,12 @@ public PhysicalHashJoin( List hashJoinConjuncts, List otherJoinConjuncts, JoinHint hint, + Optional markJoinSlotReference, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - this(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, Optional.empty(), logicalProperties, leftChild, - rightChild); + this(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, + Optional.empty(), logicalProperties, leftChild, rightChild); } /** @@ -66,10 +68,11 @@ public PhysicalHashJoin( List hashJoinConjuncts, List otherJoinConjuncts, JoinHint hint, + Optional markJoinSlotReference, Optional groupExpression, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - super(PlanType.PHYSICAL_HASH_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + super(PlanType.PHYSICAL_HASH_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, groupExpression, logicalProperties, leftChild, rightChild); } @@ -84,13 +87,14 @@ public PhysicalHashJoin( List hashJoinConjuncts, List otherJoinConjuncts, JoinHint hint, + Optional markJoinSlotReference, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - super(PlanType.PHYSICAL_HASH_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + super(PlanType.PHYSICAL_HASH_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, groupExpression, logicalProperties, physicalProperties, statsDeriveResult, leftChild, rightChild); } @@ -104,6 +108,8 @@ public String toString() { List args = Lists.newArrayList("type", joinType, "hashJoinCondition", hashJoinConjuncts, "otherJoinCondition", otherJoinConjuncts, + "isMarkJoin", markJoinSlotReference.isPresent(), + "MarkJoinSlotReference", markJoinSlotReference.isPresent() ? markJoinSlotReference.get() : "empty", "stats", statsDeriveResult); if (hint != JoinHint.NONE) { args.add("hint"); @@ -115,28 +121,28 @@ public String toString() { @Override public PhysicalHashJoin withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, getLogicalProperties(), children.get(0), children.get(1)); } @Override public PhysicalHashJoin withGroupExpression( Optional groupExpression) { - return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, groupExpression, getLogicalProperties(), left(), right()); } @Override public PhysicalHashJoin withLogicalProperties( Optional logicalProperties) { - return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, Optional.empty(), logicalProperties.get(), left(), right()); } @Override public PhysicalHashJoin withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) { - return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, + return new PhysicalHashJoin<>(joinType, hashJoinConjuncts, otherJoinConjuncts, hint, markJoinSlotReference, Optional.empty(), getLogicalProperties(), physicalProperties, statsDeriveResult, left(), right()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java index 525f40ebe750c4..808744c05afa41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalNestedLoopJoin.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; import org.apache.doris.nereids.trees.plans.JoinHint; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; @@ -46,11 +47,12 @@ public PhysicalNestedLoopJoin( JoinType joinType, List hashJoinConjuncts, List otherJoinConjuncts, + Optional markJoinSlotReference, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { - this(joinType, hashJoinConjuncts, otherJoinConjuncts, Optional.empty(), logicalProperties, leftChild, - rightChild); + this(joinType, hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, + Optional.empty(), logicalProperties, leftChild, rightChild); } /** @@ -63,12 +65,13 @@ public PhysicalNestedLoopJoin( JoinType joinType, List hashJoinConjuncts, List otherJoinConjuncts, + Optional markJoinSlotReference, Optional groupExpression, LogicalProperties logicalProperties, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { super(PlanType.PHYSICAL_NESTED_LOOP_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, // nested loop join ignores join hints. - JoinHint.NONE, + JoinHint.NONE, markJoinSlotReference, groupExpression, logicalProperties, leftChild, rightChild); } @@ -82,6 +85,7 @@ public PhysicalNestedLoopJoin( JoinType joinType, List hashJoinConjuncts, List otherJoinConjuncts, + Optional markJoinSlotReference, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, @@ -90,7 +94,7 @@ public PhysicalNestedLoopJoin( RIGHT_CHILD_TYPE rightChild) { super(PlanType.PHYSICAL_NESTED_LOOP_JOIN, joinType, hashJoinConjuncts, otherJoinConjuncts, // nested loop join ignores join hints. - JoinHint.NONE, + JoinHint.NONE, markJoinSlotReference, groupExpression, logicalProperties, physicalProperties, statsDeriveResult, leftChild, rightChild); } @@ -104,7 +108,9 @@ public String toString() { // TODO: Maybe we could pull up this to the abstract class in the future. return Utils.toSqlString("PhysicalNestedLoopJoin", "type", joinType, - "otherJoinCondition", otherJoinConjuncts + "otherJoinCondition", otherJoinConjuncts, + "isMarkJoin", markJoinSlotReference.isPresent(), + "markJoinSlotReference", markJoinSlotReference.isPresent() ? markJoinSlotReference.get() : "empty" ); } @@ -112,21 +118,23 @@ public String toString() { public PhysicalNestedLoopJoin withChildren(List children) { Preconditions.checkArgument(children.size() == 2); return new PhysicalNestedLoopJoin<>(joinType, - hashJoinConjuncts, otherJoinConjuncts, getLogicalProperties(), children.get(0), children.get(1)); + hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, + getLogicalProperties(), children.get(0), children.get(1)); } @Override public PhysicalNestedLoopJoin withGroupExpression( Optional groupExpression) { return new PhysicalNestedLoopJoin<>(joinType, - hashJoinConjuncts, otherJoinConjuncts, groupExpression, getLogicalProperties(), left(), right()); + hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, + groupExpression, getLogicalProperties(), left(), right()); } @Override public PhysicalNestedLoopJoin withLogicalProperties( Optional logicalProperties) { return new PhysicalNestedLoopJoin<>(joinType, - hashJoinConjuncts, otherJoinConjuncts, Optional.empty(), + hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, Optional.empty(), logicalProperties.get(), left(), right()); } @@ -134,7 +142,7 @@ public PhysicalNestedLoopJoin withLogicalProp public PhysicalNestedLoopJoin withPhysicalPropertiesAndStats( PhysicalProperties physicalProperties, StatsDeriveResult statsDeriveResult) { return new PhysicalNestedLoopJoin<>(joinType, - hashJoinConjuncts, otherJoinConjuncts, Optional.empty(), + hashJoinConjuncts, otherJoinConjuncts, markJoinSlotReference, Optional.empty(), getLogicalProperties(), physicalProperties, statsDeriveResult, left(), right()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java index e963d6131aa553..2188090df3343c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Not; import org.apache.doris.nereids.trees.expressions.SlotReference; @@ -208,10 +209,14 @@ private static List collectCorrelatedSlotsFromChildren( if (binaryExpression.left().anyMatch(correlatedSlots::contains)) { if (binaryExpression.right() instanceof SlotReference) { slots.add(binaryExpression.right()); + } else if (binaryExpression.right() instanceof Cast) { + slots.add(((Cast) binaryExpression.right()).child()); } } else { if (binaryExpression.left() instanceof SlotReference) { slots.add(binaryExpression.left()); + } else if (binaryExpression.left() instanceof Cast) { + slots.add(((Cast) binaryExpression.left()).child()); } } return slots; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index 0557e30b2613ad..1bbdae9d09264b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -127,8 +127,8 @@ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, TableRef inne */ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator joinOp, List eqJoinConjuncts, List otherJoinConjuncts, List srcToOutputList, - TupleDescriptor intermediateTuple, TupleDescriptor outputTuple) { - super(id, "HASH JOIN", StatisticalType.HASH_JOIN_NODE, joinOp); + TupleDescriptor intermediateTuple, TupleDescriptor outputTuple, boolean isMarkJoin) { + super(id, "HASH JOIN", StatisticalType.HASH_JOIN_NODE, joinOp, isMarkJoin); Preconditions.checkArgument(eqJoinConjuncts != null && !eqJoinConjuncts.isEmpty()); Preconditions.checkArgument(otherJoinConjuncts != null); tblRefIds.addAll(outer.getTblRefIds()); @@ -758,7 +758,6 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve StringBuilder output = new StringBuilder().append(detailPrefix).append("join op: ").append(joinOp.toString()).append("(") .append(distrModeStr).append(")").append("[").append(colocateReason).append("]\n"); - output.append(detailPrefix).append("is mark: ").append(isMarkJoin()).append("\n"); if (detailLevel == TExplainLevel.BRIEF) { output.append(detailPrefix).append( String.format("cardinality=%,d", cardinality)).append("\n"); @@ -809,6 +808,9 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve } output.append("\n"); } + if (detailLevel == TExplainLevel.VERBOSE) { + output.append(detailPrefix).append("isMarkJoin: ").append(isMarkJoin()).append("\n"); + } return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java index 5d810333621169..10e74441903f0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java @@ -58,6 +58,7 @@ public abstract class JoinNodeBase extends PlanNode { protected final TableRef innerRef; protected final JoinOperator joinOp; + protected final boolean isMark; protected TupleDescriptor vOutputTupleDesc; protected ExprSubstitutionMap vSrcToOutputSMap; protected List vIntermediateTupleDescList; @@ -85,10 +86,11 @@ public JoinNodeBase(PlanNodeId id, String planNodeName, StatisticalType statisti } else if (joinOp.equals(JoinOperator.RIGHT_OUTER_JOIN)) { nullableTupleIds.addAll(outer.getTupleIds()); } + this.isMark = this.innerRef != null && innerRef.isMark(); } public boolean isMarkJoin() { - return innerRef != null && innerRef.isMark(); + return isMark; } public JoinOperator getJoinOp() { @@ -474,10 +476,12 @@ public void finalize(Analyzer analyzer) throws UserException { /** * Only for Nereids. */ - public JoinNodeBase(PlanNodeId id, String planNodeName, StatisticalType statisticalType, JoinOperator joinOp) { + public JoinNodeBase(PlanNodeId id, String planNodeName, + StatisticalType statisticalType, JoinOperator joinOp, boolean isMark) { super(id, planNodeName, statisticalType); this.innerRef = null; this.joinOp = joinOp; + this.isMark = isMark; } public TableRef getInnerRef() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java index d49c06ce941a94..33611ff64e0aae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java @@ -92,8 +92,8 @@ protected Pair needToCopyRightAndLeft() { */ public NestedLoopJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, List tupleIds, JoinOperator joinOperator, List srcToOutputList, TupleDescriptor intermediateTuple, - TupleDescriptor outputTuple) { - super(id, "NESTED LOOP JOIN", StatisticalType.NESTED_LOOP_JOIN_NODE, joinOperator); + TupleDescriptor outputTuple, boolean isMarkJoin) { + super(id, "NESTED LOOP JOIN", StatisticalType.NESTED_LOOP_JOIN_NODE, joinOperator, isMarkJoin); this.tupleIds.addAll(tupleIds); children.add(outer); children.add(inner); @@ -228,7 +228,6 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve StringBuilder output = new StringBuilder().append(detailPrefix).append("join op: ").append(joinOp.toString()).append("(") .append(distrModeStr).append(")\n"); - output.append(detailPrefix).append("is mark: ").append(isMarkJoin()).append("\n"); if (detailLevel == TExplainLevel.BRIEF) { output.append(detailPrefix).append( @@ -267,6 +266,9 @@ public String getNodeExplainString(String detailPrefix, TExplainLevel detailLeve } output.append("\n"); } + if (detailLevel == TExplainLevel.VERBOSE) { + output.append(detailPrefix).append("isMarkJoin: ").append(isMarkJoin()).append("\n"); + } return output.toString(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java index 6d7b2413ffae8b..5ac1e8cc2f1fc0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriverTest.java @@ -66,6 +66,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; @SuppressWarnings("unused") @@ -114,7 +115,7 @@ public void testColocateJoin() { }; PhysicalHashJoin join = new PhysicalHashJoin<>(JoinType.RIGHT_OUTER_JOIN, - ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, + ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); @@ -164,7 +165,7 @@ Pair, List> getOnClauseUsedSlots( new SlotReference(new ExprId(0), "left", IntegerType.INSTANCE, false, Collections.emptyList()), new SlotReference(new ExprId(2), "right", IntegerType.INSTANCE, false, Collections.emptyList()))), - ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, groupPlan, groupPlan); + ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); Map leftMap = Maps.newHashMap(); @@ -210,7 +211,7 @@ Pair, List> getOnClauseUsedSlots( new SlotReference(new ExprId(0), "left", IntegerType.INSTANCE, false, Collections.emptyList()), new SlotReference(new ExprId(2), "right", IntegerType.INSTANCE, false, Collections.emptyList()))), - ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, groupPlan, groupPlan); + ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); Map leftMap = Maps.newHashMap(); @@ -247,7 +248,7 @@ Pair, List> getOnClauseUsedSlots( @Test public void testNestedLoopJoin() { PhysicalNestedLoopJoin join = new PhysicalNestedLoopJoin<>(JoinType.CROSS_JOIN, - ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, logicalProperties, groupPlan, + ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java index 2639dcfa18c046..57dddbb3df3268 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/RequestPropertyDeriverTest.java @@ -52,6 +52,7 @@ import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; @SuppressWarnings("unused") public class RequestPropertyDeriverTest { @@ -79,7 +80,7 @@ public void setUp() { @Test public void testNestedLoopJoin() { PhysicalNestedLoopJoin join = new PhysicalNestedLoopJoin<>(JoinType.CROSS_JOIN, - ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, logicalProperties, groupPlan, + ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); @@ -103,7 +104,7 @@ Pair, List> getOnClauseUsedSlots( }; PhysicalHashJoin join = new PhysicalHashJoin<>(JoinType.RIGHT_OUTER_JOIN, - ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, + ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); @@ -130,7 +131,7 @@ Pair, List> getOnClauseUsedSlots( }; PhysicalHashJoin join = new PhysicalHashJoin<>(JoinType.INNER_JOIN, - ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, + ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, groupPlan, groupPlan); GroupExpression groupExpression = new GroupExpression(join); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java index b23b28b1c82a6c..9e3dd545fa397f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.LessThan; import org.apache.doris.nereids.trees.expressions.NamedExpressionUtil; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.agg.Max; @@ -219,9 +220,14 @@ public void testWhereSql2AfterScalarToJoin() { logicalJoin( any(), logicalAggregate() - ).when(FieldChecker.check("joinType", JoinType.LEFT_OUTER_JOIN)) + ).when(FieldChecker.check("joinType", JoinType.LEFT_SEMI_JOIN)) .when(FieldChecker.check("otherJoinConjuncts", ImmutableList.of(new EqualTo( + new SlotReference(new ExprId(0), "k1", BigIntType.INSTANCE, true, + ImmutableList.of("default_cluster:test", "t6")), + new SlotReference(new ExprId(7), "sum(k3)", BigIntType.INSTANCE, true, + ImmutableList.of()) + ), new EqualTo( new SlotReference(new ExprId(6), "v2", BigIntType.INSTANCE, true, ImmutableList.of("default_cluster:test", "t7")), new SlotReference(new ExprId(1), "k2", BigIntType.INSTANCE, true, @@ -473,19 +479,23 @@ public void testSql10AfterScalarToJoin() { .applyBottomUp(new UnCorrelatedApplyAggregateFilter()) .applyBottomUp(new ScalarApplyToJoin()) .matches( - logicalJoin( - any(), - logicalAggregate( - logicalProject() - ) - ) - .when(j -> j.getJoinType().equals(JoinType.LEFT_OUTER_JOIN)) - .when(j -> j.getOtherJoinConjuncts().equals(ImmutableList.of( - new EqualTo(new SlotReference(new ExprId(1), "k2", BigIntType.INSTANCE, true, - ImmutableList.of("default_cluster:test", "t6")), - new SlotReference(new ExprId(6), "v2", BigIntType.INSTANCE, true, - ImmutableList.of("default_cluster:test", "t7"))) - ))) + logicalJoin( + any(), + logicalAggregate( + logicalProject() + ) + ) + .when(j -> j.getJoinType().equals(JoinType.LEFT_SEMI_JOIN)) + .when(j -> j.getOtherJoinConjuncts().equals(ImmutableList.of( + new LessThan(new SlotReference(new ExprId(0), "k1", BigIntType.INSTANCE, true, + ImmutableList.of("default_cluster:test", "t6")), + new SlotReference(new ExprId(8), "max(aa)", BigIntType.INSTANCE, true, + ImmutableList.of())), + new EqualTo(new SlotReference(new ExprId(1), "k2", BigIntType.INSTANCE, true, + ImmutableList.of("default_cluster:test", "t6")), + new SlotReference(new ExprId(6), "v2", BigIntType.INSTANCE, true, + ImmutableList.of("default_cluster:test", "t7"))) + ))) ); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoinTest.java index 42d34f2cd0b405..020e1d79d6fed1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/FindHashConditionForJoinTest.java @@ -41,6 +41,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * initial plan: @@ -75,7 +76,7 @@ public void testFindHashCondition() { Expression less = new LessThan(scoreId, studentId); List expr = ImmutableList.of(eq1, eq2, eq3, or, less); LogicalJoin join = new LogicalJoin<>(JoinType.INNER_JOIN, new ArrayList<>(), - expr, JoinHint.NONE, student, score); + expr, JoinHint.NONE, Optional.empty(), student, score); CascadesContext context = MemoTestUtils.createCascadesContext(join); List rules = Lists.newArrayList(new FindHashConditionForJoin().build()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherConditionTest.java index 3fdea34cd49672..48dbbb76218302 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherConditionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownJoinOtherConditionTest.java @@ -43,6 +43,7 @@ import org.junit.jupiter.api.TestInstance; import java.util.List; +import java.util.Optional; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class PushdownJoinOtherConditionTest { @@ -85,7 +86,7 @@ private void oneSide(JoinType joinType, boolean testRight) { right = rStudent; } - Plan join = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, condition, JoinHint.NONE, left, right); + Plan join = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, condition, JoinHint.NONE, Optional.empty(), left, right); Plan root = new LogicalProject<>(Lists.newArrayList(), join); Memo memo = rewrite(root); @@ -125,7 +126,7 @@ private void bothSideToBothSide(JoinType joinType) { Expression rightSide = new GreaterThan(rScore.getOutput().get(2), Literal.of(60)); List condition = ImmutableList.of(leftSide, rightSide); - Plan join = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, condition, JoinHint.NONE, rStudent, + Plan join = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, condition, JoinHint.NONE, Optional.empty(), rStudent, rScore); Plan root = new LogicalProject<>(Lists.newArrayList(), join); @@ -168,7 +169,7 @@ private void bothSideToOneSide(JoinType joinType, boolean testRight) { right = rStudent; } - Plan join = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, condition, JoinHint.NONE, left, right); + Plan join = new LogicalJoin<>(joinType, ExpressionUtils.EMPTY_CONDITION, condition, JoinHint.NONE, Optional.empty(), left, right); Plan root = new LogicalProject<>(Lists.newArrayList(), join); Memo memo = rewrite(root); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/MarkJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/MarkJoinTest.java new file mode 100644 index 00000000000000..b3d34a3f590a7a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/MarkJoinTest.java @@ -0,0 +1,249 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Test; + +public class MarkJoinTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + useDatabase("test"); + + createTable("CREATE TABLE `test_sq_dj1` (\n" + + " `c1` bigint(20) NULL,\n" + + " `c2` bigint(20) NULL,\n" + + " `c3` bigint(20) not NULL,\n" + + " `k4` bigint(20) not NULL,\n" + + " `k5` bigint(20) NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + + createTable("CREATE TABLE `test_sq_dj2` (\n" + + " `c1` bigint(20) NULL,\n" + + " `c2` bigint(20) NULL,\n" + + " `c3` bigint(20) not NULL,\n" + + " `k4` bigint(20) not NULL,\n" + + " `k5` bigint(20) NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`c1`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`c2`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + } + + // grouping sets + // grouping + @Test + public void test1() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 IN (SELECT c1 FROM test_sq_dj2) OR c1 < 10;"); + } + + @Test + public void test2() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 > (SELECT AVG(c1) FROM test_sq_dj2) OR c1 < 10;"); + } + + @Test + public void test2_1() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 > (SELECT AVG(c1) FROM test_sq_dj2 where test_sq_dj1.c1 = test_sq_dj2.c1);"); + } + + @Test + public void test2_2() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 > (SELECT AVG(c1) FROM test_sq_dj2 where test_sq_dj1.c1 = test_sq_dj2.c1) and c1 = 10;"); + } + + @Test + public void test2_3() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 = (SELECT AVG(c1) FROM test_sq_dj2 where test_sq_dj1.c1 = test_sq_dj2.c1) and c1 = 10;"); + } + + @Test + public void test3() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE EXISTS (SELECT c1 FROM test_sq_dj2 WHERE c1 = 10) OR c1 < 10"); + } + + @Test + public void test4() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 left semi join test_sq_dj2 on test_sq_dj1.c1 = test_sq_dj2.c1 WHERE c1 IN (SELECT c1 FROM test_sq_dj2) OR c1 < 10"); + } + + @Test + public void test5() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 IN (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1) OR c1 < 10"); + } + + /* + // Not support binaryOperator children at least one is in or exists subquery + @Test + public void test6() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE (c1 IN (SELECT c1 FROM test_sq_dj2)) != true"); + }*/ + + /* + // Not support binaryOperator children at least one is in or exists subquery + @Test + public void test7() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE (c1 IN (SELECT c1 FROM test_sq_dj2) OR c1 < 10) != true"); + }*/ + + @Test + public void test8() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 IN (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR c1 < (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR exists (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)"); + } + + @Test + public void test9() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 IN (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR c1 < (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1) " + + " AND c1 < (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1) " + + " AND EXISTS (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)"); + } + + @Test + public void test10() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE (c1 IN (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR c1 < (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)) " + + " AND EXISTS (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)"); + } + + @Test + public void test11() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 != (SELECT sum(c1) FROM test_sq_dj2) OR c1 < 10;"); + } + + @Test + public void test12() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 != (SELECT sum(c1) FROM test_sq_dj2) and c1 = 1 OR c1 < 10;"); + } + + @Test + public void test13() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE (c1 != (SELECT sum(c1) FROM test_sq_dj2) and c1 = 1 OR c1 < 10) and c1 = 10 and c1 = 15;"); + } + + @Test + public void test14() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT CASE\n" + + " WHEN (\n" + + " SELECT COUNT(*) / 2\n" + + " FROM test_sq_dj1\n" + + " ) > c1 THEN (\n" + + " SELECT AVG(c1)\n" + + " FROM test_sq_dj1\n" + + " )\n" + + " ELSE (\n" + + " SELECT SUM(c2)\n" + + " FROM test_sq_dj1\n" + + " )\n" + + " END AS kk4\n" + + " FROM test_sq_dj1 ;"); + } + + @Test + public void test14_1() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT CASE\n" + + " WHEN exists (\n" + + " SELECT COUNT(*) / 2\n" + + " FROM test_sq_dj1\n" + + " ) THEN (\n" + + " SELECT AVG(c1)\n" + + " FROM test_sq_dj1\n" + + " )\n" + + " ELSE (\n" + + " SELECT SUM(c2)\n" + + " FROM test_sq_dj1\n" + + " )\n" + + " END AS kk4\n" + + " FROM test_sq_dj1 ;"); + } + + @Test + public void test15() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 != (SELECT sum(c1) FROM test_sq_dj2 where test_sq_dj1.c1 = test_sq_dj2.c1) and c1 = 10 and c1 = 15;"); + } + + @Test + public void test16() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 IN (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR c1 < (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR exists (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)"); + } + + @Test + public void test17() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE ((c1 != (SELECT sum(c1) FROM test_sq_dj2) and c1 = 1 OR c1 < 10) and c1 = 10 and c1 = 15)" + + " and (c1 IN (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR c1 < (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1))" + + " and exists (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1);"); + } + + @Test + public void test18() { + PlanChecker.from(connectContext) + .checkPlannerResult("select * from test_sq_dj1 where test_sq_dj1.c1 != (select sum(c1) from test_sq_dj2 where test_sq_dj2.c3 = test_sq_dj1.c3) or c1 > 10"); + } + + @Test + public void test19() { + PlanChecker.from(connectContext) + .checkPlannerResult("SELECT * FROM test_sq_dj1 WHERE c1 IN (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " OR c1 < (SELECT sum(c1) FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)" + + " AND exists (SELECT c1 FROM test_sq_dj2 WHERE test_sq_dj1.c1 = test_sq_dj2.c1)"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java index 09048737a4b22d..5f7b8bac70352c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java @@ -228,20 +228,20 @@ public void testPhysicalJoin(@Mocked Plan left, @Mocked Plan right, @Mocked Logi Lists.newArrayList(new EqualTo( new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList()), new SlotReference(new ExprId(1), "b", BigIntType.INSTANCE, true, Lists.newArrayList()))), - ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, left, right); + ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, left, right); PhysicalHashJoin expected = new PhysicalHashJoin<>(JoinType.INNER_JOIN, Lists.newArrayList(new EqualTo( new SlotReference(new ExprId(0), "a", BigIntType.INSTANCE, true, Lists.newArrayList()), new SlotReference(new ExprId(1), "b", BigIntType.INSTANCE, true, Lists.newArrayList()))), - ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, left, right); + ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, left, right); Assertions.assertEquals(expected, actual); PhysicalHashJoin unexpected = new PhysicalHashJoin<>(JoinType.INNER_JOIN, Lists.newArrayList(new EqualTo( new SlotReference(new ExprId(2), "a", BigIntType.INSTANCE, false, Lists.newArrayList()), new SlotReference(new ExprId(3), "b", BigIntType.INSTANCE, true, Lists.newArrayList()))), - ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, logicalProperties, left, right); + ExpressionUtils.EMPTY_CONDITION, JoinHint.NONE, Optional.empty(), logicalProperties, left, right); Assertions.assertNotEquals(unexpected, actual); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index be62fc278bed24..2017fc29b9a0a9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -74,7 +74,7 @@ public void testLogicalJoin(@Mocked Plan left, @Mocked Plan right) { new SlotReference(new ExprId(1), "b", BigIntType.INSTANCE, true, Lists.newArrayList()))), left, right); Assertions.assertTrue(plan.toString().matches( - "LogicalJoin \\( type=INNER_JOIN, hashJoinConjuncts=\\[\\(a#\\d+ = b#\\d+\\)], otherJoinConjuncts=\\[] \\)")); + "LogicalJoin \\( type=INNER_JOIN, markJoinSlotReference=Optional.empty, hashJoinConjuncts=\\[\\(a#\\d+ = b#\\d+\\)], otherJoinConjuncts=\\[] \\)")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/LogicalPlanBuilder.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/LogicalPlanBuilder.java index 2bca62bab13791..43177698bc5391 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/LogicalPlanBuilder.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/LogicalPlanBuilder.java @@ -42,6 +42,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -113,7 +114,7 @@ public LogicalPlanBuilder join(LogicalPlan right, JoinType joinType, public LogicalPlanBuilder join(LogicalPlan right, JoinType joinType, List hashJoinConjuncts, List otherJoinConjucts) { LogicalJoin join = new LogicalJoin<>(joinType, hashJoinConjuncts, otherJoinConjucts, - JoinHint.NONE, this.plan, right); + JoinHint.NONE, Optional.empty(), this.plan, right); return from(join); } diff --git a/regression-test/data/nereids_syntax_p0/sub_query_correlated.out b/regression-test/data/nereids_syntax_p0/sub_query_correlated.out index 42a9bb5907b759..80640f7372806c 100644 --- a/regression-test/data/nereids_syntax_p0/sub_query_correlated.out +++ b/regression-test/data/nereids_syntax_p0/sub_query_correlated.out @@ -208,3 +208,150 @@ 20.0 20.0 +-- !in -- +1 2 +1 3 +2 4 +2 5 +3 3 +3 4 + +-- !scalar -- +1 2 +1 3 +2 4 +2 5 +20 2 +22 3 +24 4 +3 3 +3 4 + +-- !exists_true -- +1 2 +1 3 +2 4 +2 5 +3 3 +3 4 + +-- !in_exists_false -- +1 2 +1 3 +2 4 +2 5 +3 3 +3 4 + +-- !hash_join_with_other_conjuncts1 -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !hash_join_with_other_conjuncts2 -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !hash_join_with_other_conjuncts3 -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !hash_join_with_other_conjuncts4 -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !same_subquery_in_conjuncts -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !two_subquery_in_one_conjuncts -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !multi_subquery_in_and_scalry -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !multi_subquery_in_and_exist -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !multi_subquery_in_and_exist_sum -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 +20 2 +22 3 +24 4 + +-- !multi_subquery_in_and_in -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !multi_subquery_scalar_and_exist -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 +20 2 +22 3 +24 4 + +-- !multi_subquery_scalar_and_scalar -- +1 3 +1 2 +2 5 +2 4 +3 4 +3 3 + +-- !multi_subquery_in_first_or_in_and_in -- +3 3 + +-- !multi_subquery_in_second_or_in_and_in -- +3 3 + +-- !multi_subquery_scalar_and_in_or_scalar_and_exists_agg -- + +-- !multi_subquery_scalar_and_in_or_scalar_and_exists -- + diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy index 40117fe13ca8cd..637e53223efccc 100644 --- a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy +++ b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy @@ -258,7 +258,7 @@ suite ("sub_query_correlated") { select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 > (select sum(sub_query_correlated_subquery3.k3) a from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 = sub_query_correlated_subquery1.k2 order by a limit 1); """ - //---------subquery with Disjunctions + //---------subquery with Disjunctions------------- qt_scalar_subquery_with_disjunctions """ SELECT DISTINCT k1 FROM sub_query_correlated_subquery1 i1 WHERE ((SELECT count(*) FROM sub_query_correlated_subquery1 WHERE ((k1 = i1.k1) AND (k2 = 2)) or ((k1 = i1.k1) AND (k2 = 1)) ) > 0); """ @@ -280,4 +280,107 @@ suite ("sub_query_correlated") { END AS kk4 FROM sub_query_correlated_subquery3 ; """ + + //---------subquery mark join() Disjunctions------------ + order_qt_in """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3) OR k1 < 10; + """ + + order_qt_scalar """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 > (SELECT AVG(k1) FROM sub_query_correlated_subquery3) OR k1 < 10; + """ + + order_qt_exists_true """ + SELECT * FROM sub_query_correlated_subquery1 WHERE EXISTS (SELECT k1 FROM sub_query_correlated_subquery3 WHERE k1 = 10) OR k1 < 10; + """ + + order_qt_in_exists_false """ + SELECT * FROM sub_query_correlated_subquery1 WHERE EXISTS (SELECT k1 FROM sub_query_correlated_subquery3 WHERE k1 > 10) OR k1 < 10; + """ + + qt_hash_join_with_other_conjuncts1 """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 > sub_query_correlated_subquery3.k3) OR k1 < 10 ORDER BY k1; + """ + + qt_hash_join_with_other_conjuncts2 """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 < sub_query_correlated_subquery3.k3) OR k1 < 10 ORDER BY k1; + """ + + qt_hash_join_with_other_conjuncts3 """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 > sub_query_correlated_subquery3.k3) OR k1 < 11 ORDER BY k1; + """ + + qt_hash_join_with_other_conjuncts4 """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 < sub_query_correlated_subquery3.k3) OR k1 < 11 ORDER BY k1; + """ + + qt_same_subquery_in_conjuncts """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3) OR k1 IN (SELECT k1 FROM sub_query_correlated_subquery3) OR k1 < 10 ORDER BY k1; + """ + + qt_two_subquery_in_one_conjuncts """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3) OR k1 IN (SELECT k3 FROM sub_query_correlated_subquery3) OR k1 < 10 ORDER BY k1; + """ + + qt_multi_subquery_in_and_scalry """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + OR k1 < (SELECT sum(k1) FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.v1) + OR k1 < 10 ORDER BY k1; + """ + + qt_multi_subquery_in_and_exist """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + OR exists (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.v1) + OR k1 < 10 ORDER BY k1; + """ + + qt_multi_subquery_in_and_exist_sum """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + OR exists (SELECT sum(k1) FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.v1) + OR k1 < 10 ORDER BY k1; + """ + + qt_multi_subquery_in_and_in """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + OR k2 in (SELECT k2 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.v1) + OR k1 < 10 ORDER BY k1; + """ + + qt_multi_subquery_scalar_and_exist """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 < (SELECT sum(k1) FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + OR exists (SELECT sum(k1) FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.v1) + OR k1 < 10 ORDER BY k1; + """ + + qt_multi_subquery_scalar_and_scalar """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 < (SELECT sum(k1) FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + OR k2 < (SELECT sum(k1) FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.v1) + OR k1 < 10 ORDER BY k1; + """ + + qt_multi_subquery_in_first_or_in_and_in """ + SELECT * FROM sub_query_correlated_subquery1 WHERE (k1 in (SELECT k2 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + or k2 in (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1)) + and k1 in (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + """ + + qt_multi_subquery_in_second_or_in_and_in """ + SELECT * FROM sub_query_correlated_subquery1 WHERE k1 in (SELECT k2 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + or k2 in (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + and k1 in (SELECT k1 FROM sub_query_correlated_subquery3 where sub_query_correlated_subquery1.k2 = sub_query_correlated_subquery3.k1) + """ + + qt_multi_subquery_scalar_and_in_or_scalar_and_exists_agg """ + SELECT * FROM sub_query_correlated_subquery1 WHERE ((k1 != (SELECT sum(k1) FROM sub_query_correlated_subquery3) and k1 = 1 OR k1 < 10) and k1 = 10 and k1 = 15) + and (k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.k1) + OR k1 < (SELECT sum(k1) FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.k1)) + and exists (SELECT sum(k1) FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.k1); + """ + + qt_multi_subquery_scalar_and_in_or_scalar_and_exists """ + SELECT * FROM sub_query_correlated_subquery1 WHERE ((k1 != (SELECT sum(k1) FROM sub_query_correlated_subquery3) and k1 = 1 OR k1 < 10) and k1 = 10 and k1 = 15) + and (k1 IN (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.k1) + OR k1 < (SELECT sum(k1) FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.k1)) + and exists (SELECT k1 FROM sub_query_correlated_subquery3 WHERE sub_query_correlated_subquery1.k1 = sub_query_correlated_subquery3.k1); + """ } diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy index 744cbc67e6e55e..4ee579f83926d7 100644 --- a/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy +++ b/regression-test/suites/nereids_syntax_p0/sub_query_diff_old_optimize.groovy @@ -184,4 +184,12 @@ suite ("sub_query_diff_old_optimize") { exception "java.sql.SQLException: errCode = 2, detailMessage = Unexpected exception: scalar subquery's correlatedPredicates's operator must be EQ" } + + test { + sql """ + SELECT * FROM sub_query_diff_old_optimize_subquery1 WHERE (k1 IN (SELECT k1 FROM sub_query_diff_old_optimize_subquery3) OR k1 < 10) != true; + """ + exception "java.sql.SQLException: errCode = 2, detailMessage = Unexpected exception: Not support binaryOperator children at least one is in or exists subquery" + + } }