From c991e3a770a77e6f1c0646bdc88b0365bf7138df Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 3 Jan 2025 12:26:26 +0800 Subject: [PATCH 1/4] fix bug --- .../doris/nereids/properties/DataTrait.java | 19 ++++++++++++++++--- .../trees/plans/logical/LogicalRepeat.java | 16 +++++++++++++++- .../eliminate_group_by_key_by_uniform.out | 10 ++++++++++ .../eliminate_group_by_key_by_uniform.groovy | 3 +++ 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java index ff4756979e450e..c16328a4edf346 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java @@ -28,6 +28,7 @@ import com.google.common.collect.Sets; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -123,6 +124,10 @@ public boolean isUniformAndHasConstValue(Slot slot) { return uniformSet.isUniformAndHasConstValue(slot); } + public Map> getSlotUniformValueMap() { + return new HashMap<>(uniformSet.slotUniformValue); + } + public Optional getUniformValue(Slot slot) { return uniformSet.slotUniformValue.get(slot); } @@ -195,6 +200,10 @@ public void addUniformSlotForOuterJoinNullableSide(DataTrait dataTrait) { uniformSet.addUniformSlotForOuterJoinNullableSide(dataTrait.uniformSet); } + public void addUniformSlotValueMap(Map> map) { + uniformSet.add(map); + } + public void addUniformSlotAndLiteral(Slot slot, Expression literal) { uniformSet.add(slot, literal); } @@ -538,13 +547,17 @@ public void add(Set slots) { } } - public void add(UniformDescription ud) { - slotUniformValue.putAll(ud.slotUniformValue); - for (Map.Entry> entry : ud.slotUniformValue.entrySet()) { + public void add(Map> map) { + slotUniformValue.putAll(map); + for (Map.Entry> entry : map.entrySet()) { add(entry.getKey(), entry.getValue().orElse(null)); } } + public void add(UniformDescription ud) { + add(ud.slotUniformValue); + } + public void add(Slot slot, Expression literal) { if (null == literal) { slotUniformValue.putIfAbsent(slot, Optional.empty()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 10695f655508af..6e3c1408d3c9af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Repeat; @@ -34,9 +35,12 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * LogicalRepeat. @@ -190,7 +194,17 @@ public void computeUnique(DataTrait.Builder builder) { @Override public void computeUniform(DataTrait.Builder builder) { - builder.addUniformSlot(child(0).getLogicalProperties().getTrait()); + DataTrait dataTrait = child(0).getLogicalProperties().getTrait(); + Set common = getCommonGroupingSetExpressions(); + Map> slotUniformValue = dataTrait.getSlotUniformValueMap(); + Map> newSlotUniformValue = new HashMap<>(); + for (Map.Entry> entry : slotUniformValue.entrySet()) { + Optional value = entry.getValue(); + if (!value.isPresent() || value.get() instanceof NullLiteral || common.contains(value.get())) { + newSlotUniformValue.put(entry.getKey(), value); + } + } + builder.addUniformSlotValueMap(newSlotUniformValue); } @Override diff --git a/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out index 94cebafa4e9aad..625b8e75488669 100644 --- a/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out +++ b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.out @@ -267,3 +267,13 @@ cherry 3 -- !right_anti_right_side -- +-- !grouping -- +\N \N +\N 1 +\N 2 +\N 3 +1 \N +1 1 +1 2 +1 3 + diff --git a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy index 4375aac316d5d1..399be417c8baf6 100644 --- a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy +++ b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by_key_by_uniform.groovy @@ -218,4 +218,7 @@ suite("eliminate_group_by_key_by_uniform") { qt_left_anti_left_side "select t1.b from test1 t1 left anti join (select * from test2 where b=105) t2 on t1.a=t2.a where t1.b=1 group by t1.b,t1.a order by 1;" qt_right_semi_right_side "select t2.b from test1 t1 right semi join (select * from test2 where b=105) t2 on t1.a=t2.a group by t2.b,t2.a order by 1;" qt_right_anti_right_side "select t2.b from test1 t1 right anti join (select * from test2 where b=105) t2 on t1.a=t2.a group by t2.b,t2.a order by 1;" + + //grouping + qt_grouping "select k, k3 from (select 1 as k, a k3, sum(b) as sum_k1 from test1 group by cube(k,a)) t group by k,k3 order by 1,2" } \ No newline at end of file From 52af8e71f377d7d5240d7139c6dc35e033174f61 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 3 Jan 2025 16:58:57 +0800 Subject: [PATCH 2/4] fix feut --- .../doris/nereids/trees/plans/logical/LogicalRepeat.java | 2 ++ .../org/apache/doris/nereids/properties/DataTraitTest.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 6e3c1408d3c9af..98a5402e6ebe88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -202,6 +202,8 @@ public void computeUniform(DataTrait.Builder builder) { Optional value = entry.getValue(); if (!value.isPresent() || value.get() instanceof NullLiteral || common.contains(value.get())) { newSlotUniformValue.put(entry.getKey(), value); + } else { + newSlotUniformValue.put(entry.getKey(), Optional.empty()); } } builder.addUniformSlotValueMap(newSlotUniformValue); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java index 04e8ec7fc3240a..fc7f2f58eceae4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java @@ -242,8 +242,8 @@ void testAgg() { .rewrite() .getPlan(); System.out.println(plan.getLogicalProperties().getTrait()); - Assertions.assertTrue(plan.getLogicalProperties() - .getTrait().isUniform(plan.getOutput().get(0))); + Assertions.assertTrue(plan.getLogicalProperties().getTrait().isUniform(plan.getOutput().get(0)) + && !plan.getLogicalProperties().getTrait().isUniformAndNotNull(plan.getOutput().get(0))); } @Test From 886c68f0dfe0bb07c11daca9f179886ce259bb43 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 3 Jan 2025 20:30:30 +0800 Subject: [PATCH 3/4] fix --- .../trees/plans/logical/LogicalRepeat.java | 19 ++----------------- .../nereids/properties/DataTraitTest.java | 3 +-- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java index 98a5402e6ebe88..005ac35cf9913e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRepeat.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; -import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.Repeat; @@ -35,12 +34,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * LogicalRepeat. @@ -194,19 +190,8 @@ public void computeUnique(DataTrait.Builder builder) { @Override public void computeUniform(DataTrait.Builder builder) { - DataTrait dataTrait = child(0).getLogicalProperties().getTrait(); - Set common = getCommonGroupingSetExpressions(); - Map> slotUniformValue = dataTrait.getSlotUniformValueMap(); - Map> newSlotUniformValue = new HashMap<>(); - for (Map.Entry> entry : slotUniformValue.entrySet()) { - Optional value = entry.getValue(); - if (!value.isPresent() || value.get() instanceof NullLiteral || common.contains(value.get())) { - newSlotUniformValue.put(entry.getKey(), value); - } else { - newSlotUniformValue.put(entry.getKey(), Optional.empty()); - } - } - builder.addUniformSlotValueMap(newSlotUniformValue); + // don't generate uniform slot + // TODO: this need be supported later } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java index fc7f2f58eceae4..246495c659d9c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/DataTraitTest.java @@ -242,8 +242,7 @@ void testAgg() { .rewrite() .getPlan(); System.out.println(plan.getLogicalProperties().getTrait()); - Assertions.assertTrue(plan.getLogicalProperties().getTrait().isUniform(plan.getOutput().get(0)) - && !plan.getLogicalProperties().getTrait().isUniformAndNotNull(plan.getOutput().get(0))); + // Assertions.assertTrue(plan.getLogicalProperties().getTrait().isUniform(plan.getOutput().get(0))); } @Test From fb11a39b7577cb19da9cc5941824a0f7e681b02d Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Fri, 3 Jan 2025 20:42:12 +0800 Subject: [PATCH 4/4] fix --- .../doris/nereids/properties/DataTrait.java | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java index c16328a4edf346..ff4756979e450e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DataTrait.java @@ -28,7 +28,6 @@ import com.google.common.collect.Sets; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -124,10 +123,6 @@ public boolean isUniformAndHasConstValue(Slot slot) { return uniformSet.isUniformAndHasConstValue(slot); } - public Map> getSlotUniformValueMap() { - return new HashMap<>(uniformSet.slotUniformValue); - } - public Optional getUniformValue(Slot slot) { return uniformSet.slotUniformValue.get(slot); } @@ -200,10 +195,6 @@ public void addUniformSlotForOuterJoinNullableSide(DataTrait dataTrait) { uniformSet.addUniformSlotForOuterJoinNullableSide(dataTrait.uniformSet); } - public void addUniformSlotValueMap(Map> map) { - uniformSet.add(map); - } - public void addUniformSlotAndLiteral(Slot slot, Expression literal) { uniformSet.add(slot, literal); } @@ -547,17 +538,13 @@ public void add(Set slots) { } } - public void add(Map> map) { - slotUniformValue.putAll(map); - for (Map.Entry> entry : map.entrySet()) { + public void add(UniformDescription ud) { + slotUniformValue.putAll(ud.slotUniformValue); + for (Map.Entry> entry : ud.slotUniformValue.entrySet()) { add(entry.getKey(), entry.getValue().orElse(null)); } } - public void add(UniformDescription ud) { - add(ud.slotUniformValue); - } - public void add(Slot slot, Expression literal) { if (null == literal) { slotUniformValue.putIfAbsent(slot, Optional.empty());