From 029e51bddf17b66535193969130ab085a3986e09 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sat, 20 Jul 2024 11:40:25 +0800 Subject: [PATCH 1/5] [Fix](nereids) fix normalize repeat alias rewrite --- .../rules/analysis/NormalizeRepeat.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java index 6465b81da30539..fa7b1c327b155d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java @@ -51,6 +51,7 @@ import org.jetbrains.annotations.NotNull; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -135,7 +136,7 @@ private static LogicalAggregate normalizeRepeat(LogicalRepeat repeat Map groupingExprMap = groupingExprContext.getNormalizeToSlotMap(); Set existsAlias = getExistsAlias(repeat, groupingExprMap); Set needToSlotsArgs = collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(repeat); - NormalizeToSlotContext argsContext = NormalizeToSlotContext.buildContext(existsAlias, needToSlotsArgs); + NormalizeToSlotContext argsContext = buildContextWithAlias(repeat, existsAlias, needToSlotsArgs); // normalize grouping sets to List> ImmutableList.Builder> normalizedGroupingSetBuilder = ImmutableList.builder(); @@ -278,6 +279,30 @@ public static NormalizeToSlotContext buildContext(Repeat repeat, return new NormalizeToSlotContext(normalizeToSlotMap); } + private static NormalizeToSlotContext buildContextWithAlias(Repeat repeat, + Set existsAliases, Collection sourceExpressions) { + Map existsAliasMap = Maps.newLinkedHashMap(); + for (Alias existsAlias : existsAliases) { + existsAliasMap.put(existsAlias.child(), existsAlias); + } + List groupingSetExpressions = ExpressionUtils.flatExpressions(repeat.getGroupingSets()); + Map normalizeToSlotMap = Maps.newLinkedHashMap(); + for (Expression expression : sourceExpressions) { + Optional pushDownTriplet; + if (groupingSetExpressions.contains(expression)) { + pushDownTriplet = toGroupingSetExpressionPushDownTriplet(expression, existsAliasMap.get(expression)); + } else { + pushDownTriplet = Optional.of( + NormalizeToSlotTriplet.toTriplet(expression, existsAliasMap.get(expression))); + } + + if (pushDownTriplet.isPresent()) { + normalizeToSlotMap.put(expression, pushDownTriplet.get()); + } + } + return new NormalizeToSlotContext(normalizeToSlotMap); + } + private static Optional toGroupingSetExpressionPushDownTriplet( Expression expression, @Nullable Alias existsAlias) { NormalizeToSlotTriplet originTriplet = NormalizeToSlotTriplet.toTriplet(expression, existsAlias); From ad9968523f6fd69cb190a79fc0023bcb1ea7c71b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Sun, 21 Jul 2024 21:40:11 +0800 Subject: [PATCH 2/5] [Fix](nereids) fix normalize repeat alias rewrite --- .../rules/analysis/NormalizeRepeat.java | 37 +-- .../doris/nereids/util/ExpressionUtils.java | 9 + .../grouping_sets/grouping_alias_test.out | 211 ++++++++++++++++++ .../grouping_sets/grouping_alias_test.groovy | 66 ++++++ 4 files changed, 306 insertions(+), 17 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out create mode 100644 regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java index fa7b1c327b155d..d7e455b4e932bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java @@ -52,6 +52,7 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -134,7 +135,7 @@ private static LogicalAggregate normalizeRepeat(LogicalRepeat repeat Set needToSlotsGroupingExpr = collectNeedToSlotGroupingExpr(repeat); NormalizeToSlotContext groupingExprContext = buildContext(repeat, needToSlotsGroupingExpr); Map groupingExprMap = groupingExprContext.getNormalizeToSlotMap(); - Set existsAlias = getExistsAlias(repeat, groupingExprMap); + Map existsAlias = getExistsAlias(repeat, groupingExprMap); Set needToSlotsArgs = collectNeedToSlotArgsOfGroupingScalarFuncAndAggFunc(repeat); NormalizeToSlotContext argsContext = buildContextWithAlias(repeat, existsAlias, needToSlotsArgs); @@ -255,9 +256,12 @@ private static Plan pushDownProject(Set pushedExprs, Plan origi /** buildContext */ public static NormalizeToSlotContext buildContext(Repeat repeat, Set sourceExpressions) { - Set aliases = ExpressionUtils.collect(repeat.getOutputExpressions(), Alias.class::isInstance); + List aliases = ExpressionUtils.collectToList(repeat.getOutputExpressions(), Alias.class::isInstance); Map existsAliasMap = Maps.newLinkedHashMap(); for (Alias existsAlias : aliases) { + if (existsAliasMap.containsKey(existsAlias.child())) { + continue; + } existsAliasMap.put(existsAlias.child(), existsAlias); } @@ -280,11 +284,7 @@ public static NormalizeToSlotContext buildContext(Repeat repeat, } private static NormalizeToSlotContext buildContextWithAlias(Repeat repeat, - Set existsAliases, Collection sourceExpressions) { - Map existsAliasMap = Maps.newLinkedHashMap(); - for (Alias existsAlias : existsAliases) { - existsAliasMap.put(existsAlias.child(), existsAlias); - } + Map existsAliasMap, Collection sourceExpressions) { List groupingSetExpressions = ExpressionUtils.flatExpressions(repeat.getGroupingSets()); Map normalizeToSlotMap = Maps.newLinkedHashMap(); for (Expression expression : sourceExpressions) { @@ -295,10 +295,8 @@ private static NormalizeToSlotContext buildContextWithAlias(Repeat normalizeToSlotMap.put(expression, normalizeToSlotTriplet)); } return new NormalizeToSlotContext(normalizeToSlotMap); } @@ -329,18 +327,23 @@ private static Expression normalizeAggFuncChildrenAndGroupingScalarFunc(Normaliz } } - private static Set getExistsAlias(LogicalRepeat repeat, + private static Map getExistsAlias(LogicalRepeat repeat, Map groupingExprMap) { - Set existsAlias = Sets.newHashSet(); - Set aliases = ExpressionUtils.collect(repeat.getOutputExpressions(), Alias.class::isInstance); - existsAlias.addAll(aliases); + Map existsAliasMap = new HashMap<>(); for (NormalizeToSlotTriplet triplet : groupingExprMap.values()) { if (triplet.pushedExpr instanceof Alias) { Alias alias = (Alias) triplet.pushedExpr; - existsAlias.add(alias); + existsAliasMap.put(triplet.originExpr, alias); + } + } + List aliases = ExpressionUtils.collectToList(repeat.getOutputExpressions(), Alias.class::isInstance); + for (Alias alias : aliases) { + if (existsAliasMap.containsKey(alias.child())) { + continue; } + existsAliasMap.put(alias.child(), alias); } - return existsAlias; + return existsAliasMap; } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index b19d4b096e27d4..2f238b68757613 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -699,6 +699,15 @@ public static Set collect(Collection expressions, return set.build(); } + public static List collectToList(Collection expressions, + Predicate> predicate) { + ImmutableList.Builder list = ImmutableList.builder(); + for (Expression expr : expressions) { + list.addAll(expr.collectToList(predicate)); + } + return list.build(); + } + /** * extract uniform slot for the given predicate, such as a = 1 and b = 2 */ diff --git a/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out b/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out new file mode 100644 index 00000000000000..45e622e55c47f2 --- /dev/null +++ b/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out @@ -0,0 +1,211 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !alias -- +1 +1 +2 +2 +3 +3 +4 +4 + +-- !alias_grouping_scalar -- +4 0 +4 0 +1 0 +1 0 +3 0 +3 0 +2 0 +2 0 + +-- !alias_agg_func -- +4 4 +4 4 +2 2 +2 2 +1 1 +1 1 +3 3 +3 3 + +-- !same_column_different_alias -- +3 3 +3 3 +1 1 +1 1 +4 4 +4 4 +2 2 +2 2 + +-- !same_column_different_alias_grouping_scalar -- +1 1 0 +1 1 0 +3 3 0 +3 3 0 +2 2 0 +2 2 0 +4 4 0 +4 4 0 + +-- !same_column_different_alias_agg_func -- +3 3 3 +3 3 3 +4 4 4 +4 4 4 +2 2 2 +2 2 2 +1 1 1 +1 1 1 + +-- !same_column_different_alias_3_column -- +1 1 1 +1 1 1 +2 2 2 +2 2 2 +4 4 4 +4 4 4 +3 3 3 +3 3 3 + +-- !same_column_different_alias_3_column_grouping_scalar -- +2 2 2 0 +2 2 2 0 +4 4 4 0 +4 4 4 0 +3 3 3 0 +3 3 3 0 +1 1 1 0 +1 1 1 0 + +-- !same_column_different_alias_3_column_agg_func -- +3 3 3 3 +3 3 3 3 +4 4 4 4 +4 4 4 4 +1 1 1 1 +1 1 1 1 +2 2 2 2 +2 2 2 2 + +-- !same_column_one_has_alias_the_other_do_not -- +2 2 +2 2 +4 4 +4 4 +3 3 +3 3 +1 1 +1 1 + +-- !same_column_one_has_alias_the_other_do_not_grouping_scalar -- +4 4 0 +4 4 0 +1 1 0 +1 1 0 +3 3 0 +3 3 0 +2 2 0 +2 2 0 + +-- !same_column_one_has_alias_the_other_do_not_agg_func -- +1 1 1 +1 1 1 +4 4 4 +4 4 4 +2 2 2 +2 2 2 +3 3 3 +3 3 3 + +-- !same_column_different_alias -- +2 2 +2 2 +5 5 +5 5 +4 4 +4 4 +3 3 +3 3 + +-- !same_column_different_alias_grouping_scalar -- +3 3 0 +3 3 0 +5 5 0 +5 5 0 +4 4 0 +4 4 0 +2 2 0 +2 2 0 + +-- !same_column_different_alias_agg_func -- +5 5 5 +5 5 5 +3 3 3 +3 3 3 +2 2 2 +2 2 2 +4 4 4 +4 4 4 + +-- !same_column_different_alias_3_column -- +3 3 3 +3 3 3 +2 2 2 +2 2 2 +5 5 5 +5 5 5 +4 4 4 +4 4 4 + +-- !same_column_different_alias_3_column_grouping_scalar -- +4 4 4 0 +4 4 4 0 +3 3 3 0 +3 3 3 0 +2 2 2 0 +2 2 2 0 +5 5 5 0 +5 5 5 0 + +-- !same_column_different_alias_3_column_agg_func -- +5 5 5 5 +5 5 5 5 +2 2 2 2 +2 2 2 2 +4 4 4 4 +4 4 4 4 +3 3 3 3 +3 3 3 3 + +-- !same_column_one_has_alias_the_other_do_not -- +3 3 +3 3 +4 4 +4 4 +5 5 +5 5 +2 2 +2 2 + +-- !same_column_one_has_alias_the_other_do_not_grouping_scalar -- +5 5 0 +5 5 0 +4 4 0 +4 4 0 +3 3 0 +3 3 0 +2 2 0 +2 2 0 + +-- !same_column_one_has_alias_the_other_do_not_agg_func -- +2 2 2 +2 2 2 +5 5 5 +5 5 5 +4 4 4 +4 4 4 +3 3 3 +3 3 3 + diff --git a/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy b/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy new file mode 100644 index 00000000000000..9cccd29c6c64e6 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("grouping_alias_test"){ + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql"drop table if exists grouping_alias_test_t" + sql """ + CREATE TABLE grouping_alias_test_t ( + id INT, + value1 INT, + value2 VARCHAR(50) + )distributed by hash(id) properties("replication_num"="1"); + """ + sql """ + INSERT INTO grouping_alias_test_t (id, value1, value2) VALUES + (1, 10, 'A'), + (2, 20, 'B'), + (3, 30, 'C'), + (4, 40, 'D'); + """ + + qt_alias "select id as c1 from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_alias_grouping_scalar "select id as c1, grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_alias_agg_func "select id as c1, sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + + qt_same_column_different_alias "select id as c1, id as c2 from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_different_alias_grouping_scalar "select id as c1, id as c2, grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_different_alias_agg_func "select id as c1, id as c2 , sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + + qt_same_column_different_alias_3_column "select id as c1, id as c2, id as c3 from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_different_alias_3_column_grouping_scalar "select id as c1, id as c2, id as c3,grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_different_alias_3_column_agg_func "select id as c1, id as c2, id as c3 ,sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + + qt_same_column_one_has_alias_the_other_do_not "select id , id as c2 from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_one_has_alias_the_other_do_not_grouping_scalar "select id , id as c2 ,grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_one_has_alias_the_other_do_not_agg_func "select id , id as c2 ,sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + + qt_same_expr_different_alias "select id+1 as c1, id+1 as c2 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_different_alias_grouping_scalar "select id+1 as c1, id+1 as c2, grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_different_alias_agg_func "select id+1 as c1, id+1 as c2 , sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + + qt_same_expr_different_alias_3_expr "select id+1 as c1, id+1 as c2, id+1 as c3 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_different_alias_3_expr_grouping_scalar "select id+1 as c1, id+1 as c2, id+1 as c3,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_different_alias_3_expr_agg_func "select id+1 as c1, id+1 as c2, id+1 as c3 ,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + + qt_same_expr_one_has_alias_the_other_do_not "select id+1 , id+1 as c2 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_one_has_alias_the_other_do_not_grouping_scalar "select id+1 , id+1 as c2 ,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_one_has_alias_the_other_do_not_agg_func "select id+1 , id+1 as c2 ,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + + +} \ No newline at end of file From 6987b73c20d7ef9a9ba596341a55b1c31334087c Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 22 Jul 2024 11:04:14 +0800 Subject: [PATCH 3/5] [Fix](nereids) fix normalize repeat alias rewrite --- .../grouping_sets/grouping_alias_test.out | 274 +++++++++++++----- .../grouping_sets/grouping_alias_test.groovy | 56 ++-- 2 files changed, 232 insertions(+), 98 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out b/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out index 45e622e55c47f2..cf5a0c66ffabca 100644 --- a/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out +++ b/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out @@ -10,202 +10,322 @@ 4 -- !alias_grouping_scalar -- -4 0 -4 0 1 0 1 0 -3 0 -3 0 2 0 2 0 +3 0 +3 0 +4 0 +4 0 -- !alias_agg_func -- -4 4 -4 4 -2 2 -2 2 1 1 1 1 +2 2 +2 2 3 3 3 3 +4 4 +4 4 -- !same_column_different_alias -- -3 3 -3 3 1 1 1 1 -4 4 -4 4 2 2 2 2 +3 3 +3 3 +4 4 +4 4 -- !same_column_different_alias_grouping_scalar -- 1 1 0 1 1 0 -3 3 0 -3 3 0 2 2 0 2 2 0 +3 3 0 +3 3 0 4 4 0 4 4 0 -- !same_column_different_alias_agg_func -- +1 1 1 +1 1 1 +2 2 2 +2 2 2 3 3 3 3 3 3 4 4 4 4 4 4 -2 2 2 -2 2 2 -1 1 1 -1 1 1 -- !same_column_different_alias_3_column -- 1 1 1 1 1 1 2 2 2 2 2 2 -4 4 4 -4 4 4 3 3 3 3 3 3 +4 4 4 +4 4 4 -- !same_column_different_alias_3_column_grouping_scalar -- +1 1 1 0 +1 1 1 0 2 2 2 0 2 2 2 0 -4 4 4 0 -4 4 4 0 3 3 3 0 3 3 3 0 -1 1 1 0 -1 1 1 0 +4 4 4 0 +4 4 4 0 -- !same_column_different_alias_3_column_agg_func -- -3 3 3 3 -3 3 3 3 -4 4 4 4 -4 4 4 4 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 +3 3 3 3 +3 3 3 3 +4 4 4 4 +4 4 4 4 -- !same_column_one_has_alias_the_other_do_not -- +1 1 +1 1 2 2 2 2 -4 4 -4 4 3 3 3 3 -1 1 -1 1 +4 4 +4 4 -- !same_column_one_has_alias_the_other_do_not_grouping_scalar -- -4 4 0 -4 4 0 1 1 0 1 1 0 -3 3 0 -3 3 0 2 2 0 2 2 0 +3 3 0 +3 3 0 +4 4 0 +4 4 0 -- !same_column_one_has_alias_the_other_do_not_agg_func -- 1 1 1 1 1 1 -4 4 4 -4 4 4 2 2 2 2 2 2 3 3 3 3 3 3 +4 4 4 +4 4 4 --- !same_column_different_alias -- +-- !alias_expr -- +2 +2 +3 +3 +4 +4 +5 +5 + +-- !alias_grouping_scalar_expr -- +2 0 +2 0 +3 0 +3 0 +4 0 +4 0 +5 0 +5 0 + +-- !alias_agg_func_expr -- 2 2 2 2 -5 5 -5 5 +3 3 +3 3 4 4 4 4 +5 5 +5 5 + +-- !same_expr_different_alias -- +2 2 +2 2 3 3 3 3 +4 4 +4 4 +5 5 +5 5 --- !same_column_different_alias_grouping_scalar -- +-- !same_expr_different_alias_grouping_scalar -- +2 2 0 +2 2 0 3 3 0 3 3 0 -5 5 0 -5 5 0 4 4 0 4 4 0 -2 2 0 -2 2 0 +5 5 0 +5 5 0 --- !same_column_different_alias_agg_func -- -5 5 5 -5 5 5 -3 3 3 -3 3 3 +-- !same_expr_different_alias_agg_func -- 2 2 2 2 2 2 +3 3 3 +3 3 3 4 4 4 4 4 4 +5 5 5 +5 5 5 --- !same_column_different_alias_3_column -- -3 3 3 -3 3 3 +-- !same_expr_different_alias_3_expr -- 2 2 2 2 2 2 -5 5 5 -5 5 5 +3 3 3 +3 3 3 4 4 4 4 4 4 +5 5 5 +5 5 5 --- !same_column_different_alias_3_column_grouping_scalar -- -4 4 4 0 -4 4 4 0 -3 3 3 0 -3 3 3 0 +-- !same_expr_different_alias_3_expr_grouping_scalar -- 2 2 2 0 2 2 2 0 +3 3 3 0 +3 3 3 0 +4 4 4 0 +4 4 4 0 5 5 5 0 5 5 5 0 --- !same_column_different_alias_3_column_agg_func -- -5 5 5 5 -5 5 5 5 +-- !same_expr_different_alias_3_expr_agg_func -- 2 2 2 2 2 2 2 2 -4 4 4 4 -4 4 4 4 3 3 3 3 3 3 3 3 +4 4 4 4 +4 4 4 4 +5 5 5 5 +5 5 5 5 --- !same_column_one_has_alias_the_other_do_not -- +-- !same_expr_one_has_alias_the_other_do_not -- +2 2 +2 2 3 3 3 3 4 4 4 4 5 5 5 5 -2 2 -2 2 --- !same_column_one_has_alias_the_other_do_not_grouping_scalar -- -5 5 0 -5 5 0 -4 4 0 -4 4 0 -3 3 0 -3 3 0 +-- !same_expr_one_has_alias_the_other_do_not_grouping_scalar -- 2 2 0 2 2 0 +3 3 0 +3 3 0 +4 4 0 +4 4 0 +5 5 0 +5 5 0 --- !same_column_one_has_alias_the_other_do_not_agg_func -- +-- !same_expr_one_has_alias_the_other_do_not_agg_func -- 2 2 2 2 2 2 +3 3 3 +3 3 3 +4 4 4 +4 4 4 5 5 5 5 5 5 + +-- !expr_without_alias -- +2 +2 +3 +3 +4 +4 +5 +5 + +-- !expr_without_alias_grouping_scalar -- +2 0 +2 0 +3 0 +3 0 +4 0 +4 0 +5 0 +5 0 + +-- !expr_without_alias_agg_func -- +2 2 +2 2 +3 3 +3 3 +4 4 +4 4 +5 5 +5 5 + +-- !same_expr_without_alias -- +2 2 +2 2 +3 3 +3 3 +4 4 +4 4 +5 5 +5 5 + +-- !same_expr_without_alias_grouping_scalar -- +2 2 0 +2 2 0 +3 3 0 +3 3 0 +4 4 0 +4 4 0 +5 5 0 +5 5 0 + +-- !same_expr_without_alias_agg_func -- +2 2 2 +2 2 2 +3 3 3 +3 3 3 4 4 4 4 4 4 +5 5 5 +5 5 5 + +-- !expr_with_or_without_alias_3_expr -- +2 2 2 +2 2 2 3 3 3 3 3 3 +4 4 4 +4 4 4 +5 5 5 +5 5 5 + +-- !expr_with_or_without_alias_3_expr_grouping_scalar -- +2 2 2 0 +2 2 2 0 +3 3 3 0 +3 3 3 0 +4 4 4 0 +4 4 4 0 +5 5 5 0 +5 5 5 0 + +-- !expr_with_or_without_alias_3_expr_agg_func -- +2 2 2 2 +2 2 2 2 +3 3 3 3 +3 3 3 3 +4 4 4 4 +4 4 4 4 +5 5 5 5 +5 5 5 5 diff --git a/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy b/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy index 9cccd29c6c64e6..ab55a74a4da1ad 100644 --- a/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy +++ b/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy @@ -34,33 +34,47 @@ suite("grouping_alias_test"){ (4, 40, 'D'); """ - qt_alias "select id as c1 from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_alias_grouping_scalar "select id as c1, grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_alias_agg_func "select id as c1, sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_alias "select id as c1 from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1;" + qt_alias_grouping_scalar "select id as c1, grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2;" + qt_alias_agg_func "select id as c1, sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2;" - qt_same_column_different_alias "select id as c1, id as c2 from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_same_column_different_alias_grouping_scalar "select id as c1, id as c2, grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_same_column_different_alias_agg_func "select id as c1, id as c2 , sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_different_alias "select id as c1, id as c2 from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2;" + qt_same_column_different_alias_grouping_scalar "select id as c1, id as c2, grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2,3;" + qt_same_column_different_alias_agg_func "select id as c1, id as c2 , sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2,3;" - qt_same_column_different_alias_3_column "select id as c1, id as c2, id as c3 from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_same_column_different_alias_3_column_grouping_scalar "select id as c1, id as c2, id as c3,grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_same_column_different_alias_3_column_agg_func "select id as c1, id as c2, id as c3 ,sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_different_alias_3_column "select id as c1, id as c2, id as c3 from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2,3;" + qt_same_column_different_alias_3_column_grouping_scalar "select id as c1, id as c2, id as c3,grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2,3,4;" + qt_same_column_different_alias_3_column_agg_func "select id as c1, id as c2, id as c3 ,sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2,3,4;" - qt_same_column_one_has_alias_the_other_do_not "select id , id as c2 from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_same_column_one_has_alias_the_other_do_not_grouping_scalar "select id , id as c2 ,grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" - qt_same_column_one_has_alias_the_other_do_not_agg_func "select id , id as c2 ,sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id));" + qt_same_column_one_has_alias_the_other_do_not "select id , id as c2 from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2;" + qt_same_column_one_has_alias_the_other_do_not_grouping_scalar "select id , id as c2 ,grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2,3;" + qt_same_column_one_has_alias_the_other_do_not_agg_func "select id , id as c2 ,sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(id)) order by 1,2,3;" - qt_same_expr_different_alias "select id+1 as c1, id+1 as c2 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" - qt_same_expr_different_alias_grouping_scalar "select id+1 as c1, id+1 as c2, grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" - qt_same_expr_different_alias_agg_func "select id+1 as c1, id+1 as c2 , sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_alias_expr "select id+1 as c1 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1;" + qt_alias_grouping_scalar_expr "select id+1 as c1, grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2;" + qt_alias_agg_func_expr "select id+1 as c1, sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2;" - qt_same_expr_different_alias_3_expr "select id+1 as c1, id+1 as c2, id+1 as c3 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" - qt_same_expr_different_alias_3_expr_grouping_scalar "select id+1 as c1, id+1 as c2, id+1 as c3,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" - qt_same_expr_different_alias_3_expr_agg_func "select id+1 as c1, id+1 as c2, id+1 as c3 ,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_different_alias "select id+1 as c1, id+1 as c2 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2;" + qt_same_expr_different_alias_grouping_scalar "select id+1 as c1, id+1 as c2, grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" + qt_same_expr_different_alias_agg_func "select id+1 as c1, id+1 as c2 , sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" - qt_same_expr_one_has_alias_the_other_do_not "select id+1 , id+1 as c2 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" - qt_same_expr_one_has_alias_the_other_do_not_grouping_scalar "select id+1 , id+1 as c2 ,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" - qt_same_expr_one_has_alias_the_other_do_not_agg_func "select id+1 , id+1 as c2 ,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1));" + qt_same_expr_different_alias_3_expr "select id+1 as c1, id+1 as c2, id+1 as c3 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" + qt_same_expr_different_alias_3_expr_grouping_scalar "select id+1 as c1, id+1 as c2, id+1 as c3,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3,4;" + qt_same_expr_different_alias_3_expr_agg_func "select id+1 as c1, id+1 as c2, id+1 as c3 ,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3,4;" + qt_same_expr_one_has_alias_the_other_do_not "select id+1 , id+1 as c2 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2;" + qt_same_expr_one_has_alias_the_other_do_not_grouping_scalar "select id+1 , id+1 as c2 ,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" + qt_same_expr_one_has_alias_the_other_do_not_agg_func "select id+1 , id+1 as c2 ,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" + qt_expr_without_alias "select id+1 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1;" + qt_expr_without_alias_grouping_scalar "select id+1, grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2;" + qt_expr_without_alias_agg_func "select id+1, sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2;" + + qt_same_expr_without_alias "select id+1, id+1 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2;" + qt_same_expr_without_alias_grouping_scalar "select id+1, id+1, grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" + qt_same_expr_without_alias_agg_func "select id+1, id+1, sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" + + qt_expr_with_or_without_alias_3_expr "select id+1, id+1, id+1 as c1 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" + qt_expr_with_or_without_alias_3_expr_grouping_scalar "select id+1, id+1, id+1 as c3,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3,4;" + qt_expr_with_or_without_alias_3_expr_agg_func "select id+1 as c1, id+1, id+1,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3,4;" } \ No newline at end of file From 025a6631e3539cff5a6100dda9127b1cdf2b4f6b Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Mon, 22 Jul 2024 12:47:14 +0800 Subject: [PATCH 4/5] [Fix](nereids) fix normalize repeat alias rewrite --- .../nereids/rules/analysis/NormalizeRepeat.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java index d7e455b4e932bd..96ea874f259422 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/NormalizeRepeat.java @@ -265,20 +265,12 @@ public static NormalizeToSlotContext buildContext(Repeat repeat, existsAliasMap.put(existsAlias.child(), existsAlias); } - List groupingSetExpressions = ExpressionUtils.flatExpressions(repeat.getGroupingSets()); Map normalizeToSlotMap = Maps.newLinkedHashMap(); for (Expression expression : sourceExpressions) { - Optional pushDownTriplet; - if (groupingSetExpressions.contains(expression)) { - pushDownTriplet = toGroupingSetExpressionPushDownTriplet(expression, existsAliasMap.get(expression)); - } else { - pushDownTriplet = Optional.of( - NormalizeToSlotTriplet.toTriplet(expression, existsAliasMap.get(expression))); - } - - if (pushDownTriplet.isPresent()) { - normalizeToSlotMap.put(expression, pushDownTriplet.get()); - } + Optional pushDownTriplet = + toGroupingSetExpressionPushDownTriplet(expression, existsAliasMap.get(expression)); + pushDownTriplet.ifPresent( + normalizeToSlotTriplet -> normalizeToSlotMap.put(expression, normalizeToSlotTriplet)); } return new NormalizeToSlotContext(normalizeToSlotMap); } From 1487abfdbe77d4f7fe5ab308c15420acd4657db2 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Tue, 23 Jul 2024 21:41:26 +0800 Subject: [PATCH 5/5] add some test --- .../grouping_sets/grouping_alias_test.out | 200 ++++++++++++++++++ .../grouping_sets/grouping_alias_test.groovy | 31 +++ 2 files changed, 231 insertions(+) diff --git a/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out b/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out index cf5a0c66ffabca..27a165a1256003 100644 --- a/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out +++ b/regression-test/data/nereids_rules_p0/grouping_sets/grouping_alias_test.out @@ -329,3 +329,203 @@ 5 5 5 5 5 5 5 5 +-- !alias_in_grouping -- +1 +1 +2 +2 +3 +3 +4 +4 + +-- !alias_grouping_scalar_in_grouping -- +1 0 +1 0 +2 0 +2 0 +3 0 +3 0 +4 0 +4 0 + +-- !alias_agg_func_in_grouping -- +1 1 +1 1 +2 2 +2 2 +3 3 +3 3 +4 4 +4 4 + +-- !same_column_different_alias_in_grouping -- +1 1 +1 1 +2 2 +2 2 +3 3 +3 3 +4 4 +4 4 + +-- !same_column_different_alias_grouping_scalar_in_grouping -- +1 1 0 +1 1 0 +2 2 0 +2 2 0 +3 3 0 +3 3 0 +4 4 0 +4 4 0 + +-- !same_column_different_alias_agg_func_in_grouping -- +1 1 1 +1 1 1 +2 2 2 +2 2 2 +3 3 3 +3 3 3 +4 4 4 +4 4 4 + +-- !same_column_different_alias_in_grouping_3_column -- +1 1 1 +1 1 1 +2 2 2 +2 2 2 +3 3 3 +3 3 3 +4 4 4 +4 4 4 + +-- !same_column_different_alias_in_grouping_3_column_grouping_scalar -- +1 1 1 0 +1 1 1 0 +2 2 2 0 +2 2 2 0 +3 3 3 0 +3 3 3 0 +4 4 4 0 +4 4 4 0 + +-- !same_column_different_alias_in_grouping_3_column_agg_func -- +1 1 1 1 +1 1 1 1 +2 2 2 2 +2 2 2 2 +3 3 3 3 +3 3 3 3 +4 4 4 4 +4 4 4 4 + +-- !same_column_one_has_alias_in_grouping_the_other_do_not -- +1 1 +1 1 +2 2 +2 2 +3 3 +3 3 +4 4 +4 4 + +-- !same_column_one_has_alias_in_grouping_the_other_do_not_grouping_scalar -- +1 1 0 +1 1 0 +2 2 0 +2 2 0 +3 3 0 +3 3 0 +4 4 0 +4 4 0 + +-- !same_column_one_has_alias_in_grouping_the_other_do_not_agg_func -- +1 1 1 +1 1 1 +2 2 2 +2 2 2 +3 3 3 +3 3 3 +4 4 4 +4 4 4 + +-- !alias_in_grouping_expr -- +2 +2 +3 +3 +4 +4 +5 +5 + +-- !alias_in_grouping_grouping_scalar_expr -- +2 0 +2 0 +3 0 +3 0 +4 0 +4 0 +5 0 +5 0 + +-- !alias_in_grouping_agg_func_expr -- +2 2 +2 2 +3 3 +3 3 +4 4 +4 4 +5 5 +5 5 + +-- !same_expr_different_alias_in_grouping -- +2 2 +2 2 +3 3 +3 3 +4 4 +4 4 +5 5 +5 5 + +-- !same_expr_different_alias_in_grouping_grouping_scalar -- +2 2 0 +2 2 0 +3 3 0 +3 3 0 +4 4 0 +4 4 0 +5 5 0 +5 5 0 + +-- !same_expr_different_alias_in_grouping_agg_func -- +2 2 2 +2 2 2 +3 3 3 +3 3 3 +4 4 4 +4 4 4 +5 5 5 +5 5 5 + +-- !same_expr_different_alias_in_grouping_3_expr -- +2 2 2 0 2 +2 2 2 0 2 +3 3 3 0 3 +3 3 3 0 3 +4 4 4 0 4 +4 4 4 0 4 +5 5 5 0 5 +5 5 5 0 5 + +-- !same_expr_one_has_alias_in_grouping_the_other_do_not -- +2 2 0 2 +2 2 0 2 +3 3 0 3 +3 3 0 3 +4 4 0 4 +4 4 0 4 +5 5 0 5 +5 5 0 5 + diff --git a/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy b/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy index ab55a74a4da1ad..ee1b9b998b4f7f 100644 --- a/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy +++ b/regression-test/suites/nereids_rules_p0/grouping_sets/grouping_alias_test.groovy @@ -77,4 +77,35 @@ suite("grouping_alias_test"){ qt_expr_with_or_without_alias_3_expr "select id+1, id+1, id+1 as c1 from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3;" qt_expr_with_or_without_alias_3_expr_grouping_scalar "select id+1, id+1, id+1 as c3,grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3,4;" qt_expr_with_or_without_alias_3_expr_agg_func "select id+1 as c1, id+1, id+1,sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(id+1)) order by 1,2,3,4;" + + qt_alias_in_grouping "select id as c1 from grouping_alias_test_t group by grouping sets((c1,value2),(id)) order by 1;" + qt_alias_grouping_scalar_in_grouping "select id as c1, grouping(id) from grouping_alias_test_t group by grouping sets((c1,value2),(id)) order by 1,2;" + qt_alias_agg_func_in_grouping "select id as c1, sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(c1)) order by 1,2;" + + // TODO: The query result of the following example is different from pg. It needs to be modified later. + qt_same_column_different_alias_in_grouping "select id as c1, id as c2 from grouping_alias_test_t group by grouping sets((c1,value2),(c2)) order by 1,2;" + qt_same_column_different_alias_grouping_scalar_in_grouping "select id as c1, id as c2, grouping(id) from grouping_alias_test_t group by grouping sets((c1,value2),(c2)) order by 1,2,3;" + qt_same_column_different_alias_agg_func_in_grouping "select id as c1, id as c2 , sum(id) from grouping_alias_test_t group by grouping sets((c1,value2),(c2)) order by 1,2,3;" + + qt_same_column_different_alias_in_grouping_3_column "select id as c1, id as c2, id as c3 from grouping_alias_test_t group by grouping sets((c1,value2),(c2,c3)) order by 1,2,3;" + qt_same_column_different_alias_in_grouping_3_column_grouping_scalar "select id as c1, id as c2, id as c3,grouping(id) from grouping_alias_test_t group by grouping sets((c1,value2),(c2,c3)) order by 1,2,3,4;" + qt_same_column_different_alias_in_grouping_3_column_agg_func "select id as c1, id as c2, id as c3 ,sum(id) from grouping_alias_test_t group by grouping sets((c1,value2),(c2,c3)) order by 1,2,3,4;" + + qt_same_column_one_has_alias_in_grouping_the_other_do_not "select id , id as c2 from grouping_alias_test_t group by grouping sets((id,value2),(c2)) order by 1,2;" + qt_same_column_one_has_alias_in_grouping_the_other_do_not_grouping_scalar "select id , id as c2 ,grouping(id) from grouping_alias_test_t group by grouping sets((id,value2),(c2)) order by 1,2,3;" + qt_same_column_one_has_alias_in_grouping_the_other_do_not_agg_func "select id , id as c2 ,sum(id) from grouping_alias_test_t group by grouping sets((id,value2),(c2)) order by 1,2,3;" + + qt_alias_in_grouping_expr "select id+1 as c1 from grouping_alias_test_t group by grouping sets((id+1,value2),(c1)) order by 1;" + qt_alias_in_grouping_grouping_scalar_expr "select id+1 as c1, grouping(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(c1)) order by 1,2;" + qt_alias_in_grouping_agg_func_expr "select id+1 as c1, sum(id+1) from grouping_alias_test_t group by grouping sets((id+1,value2),(c1)) order by 1,2;" + + qt_same_expr_different_alias_in_grouping "select id+1 as c1, id+1 as c2 from grouping_alias_test_t group by grouping sets((c1,value2),(id+1,c2)) order by 1,2;" + qt_same_expr_different_alias_in_grouping_grouping_scalar "select id+1 as c1, id+1 as c2, grouping(id+1) from grouping_alias_test_t group by grouping sets((c1,value2),(id+1,c2)) order by 1,2,3;" + qt_same_expr_different_alias_in_grouping_agg_func "select id+1 as c1, id+1 as c2 , sum(id+1) from grouping_alias_test_t group by grouping sets((c1,value2),(c2)) order by 1,2,3;" + + qt_same_expr_different_alias_in_grouping_3_expr "select id+1 as c1, id+1 as c2, id+1 as c3,grouping(id+1),sum(id+1) as c3 from grouping_alias_test_t group by grouping sets((c1,value2),(c2,c3,id+1)) order by 1,2,3;" + + qt_same_expr_one_has_alias_in_grouping_the_other_do_not "select id+1 , id+1 as c2,grouping(id+1),sum(id+1) from grouping_alias_test_t group by grouping sets((c2,value2),(id+1)) order by 1,2;" + + } \ No newline at end of file