From 9cc325ba5720aa08bfabb8df001bab5569115ce8 Mon Sep 17 00:00:00 2001 From: starocean999 <12095047@qq.com> Date: Fri, 10 Nov 2023 14:22:17 +0800 Subject: [PATCH 1/3] [enhancement](nereids)make error message more readable when bind logicalRepeat node --- .../rules/analysis/BindExpression.java | 15 ++++++++ .../grouping_sets/test_grouping_sets.groovy | 38 ++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 0d36ca31c32d79..2608a554fc94d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; import org.apache.doris.nereids.trees.expressions.functions.table.TableValuedFunction; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; @@ -74,6 +75,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.UsingJoin; import org.apache.doris.nereids.trees.plans.visitor.InferPlanOutputAlias; +import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; @@ -386,6 +388,19 @@ protected boolean condition(Rule rule, Plan plan) { .collect(ImmutableList.toImmutableList()); List newOutput = adjustNullableForRepeat(groupingSets, output); groupingSets.forEach(list -> checkIfOutputAliasNameDuplicatedForGroupBy(list, newOutput)); + + // check all GroupingScalarFunction inputSlots must be from groupingExprs + Set groupingExprs = groupingSets.stream() + .flatMap(Collection::stream).map(expr -> expr.getInputSlots()) + .flatMap(Collection::stream).collect(Collectors.toSet()); + Set groupingScalarFunctions = ExpressionUtils + .collect(newOutput, GroupingScalarFunction.class::isInstance); + for (GroupingScalarFunction function : groupingScalarFunctions) { + if (!groupingExprs.containsAll(function.getInputSlots())) { + throw new AnalysisException("Column in " + function.getName() + + " does not exist in GROUP BY clause."); + } + } return repeat.withGroupSetsAndOutput(groupingSets, newOutput); }) ), diff --git a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy index e27b067f9abf8c..f2994233087925 100644 --- a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy +++ b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy @@ -46,7 +46,7 @@ suite("test_grouping_sets", "p0") { test { sql """ - SELECT k1, k2, SUM(k3) FROM test_query_db.test + SELECT /*+ SET_VAR(enable_nereids_planner=false) */ k1, k2, SUM(k3) FROM test_query_db.test GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ( ), (k3) ) order by k1, k2 """ exception "errCode = 2, detailMessage = column: `k3` cannot both in select list and aggregate functions" @@ -54,12 +54,28 @@ suite("test_grouping_sets", "p0") { test { sql """ - SELECT k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test + SELECT /*+ SET_VAR(enable_nereids_planner=true) */ k1, k2, SUM(k3) FROM test_query_db.test + GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ( ), (k3) ) order by k1, k2 + """ + exception "errCode = 2, detailMessage = column: k3 cannot both in select list and aggregate functions" + } + + test { + sql """ + SELECT /*+ SET_VAR(enable_nereids_planner=false) */ k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ( ), (k3) ) order by k1, k2 """ exception "errCode = 2, detailMessage = column: `k3` cannot both in select list and aggregate functions" } + test { + sql """ + SELECT /*+ SET_VAR(enable_nereids_planner=true) */ k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test + GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ( ), (k3) ) order by k1, k2 + """ + exception "errCode = 2, detailMessage = column: k3 cannot both in select list and aggregate functions" + } + qt_select7 """ select k1,k2,sum(k3) from test_query_db.test where 1 = 2 group by grouping sets((k1), (k1,k2)) """ qt_select8 """ WITH dt AS @@ -164,9 +180,15 @@ suite("test_grouping_sets", "p0") { qt_select19 """SELECT k1 ,GROUPING(k1) FROM test_query_db.test GROUP BY ROLLUP (k1) ORDER BY k1""" qt_select20 """SELECT k1 ,GROUPING(k1) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1""" test { - sql "SELECT k1 ,GROUPING(k2) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1" + sql "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ k1 ,GROUPING(k2) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1" exception "Column `k2` in GROUP_ID() does not exist in GROUP BY clause" } + + test { + sql "SELECT /*+ SET_VAR(enable_nereids_planner=true) */ k1 ,GROUPING(k2) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1" + exception "Column in Grouping does not exist in GROUP BY clause" + } + // test grouping sets id contain null data sql """drop table if exists test_query_db.test_grouping_sets_id_null""" sql """create table if not exists test_query_db.test_grouping_sets_id_null like test_query_db.test""" @@ -192,8 +214,6 @@ suite("test_grouping_sets", "p0") { """ sql """drop table if exists test_query_db.test_grouping_sets_id_null""" // test grouping sets shoot rollup - // because nereids cannot support rollup correctly forbid it temporary - sql """set enable_nereids_planner=false""" sql "drop table if exists test_query_db.test_grouping_sets_rollup" sql """ create table if not exists test_query_db.test_grouping_sets_rollup( @@ -232,8 +252,14 @@ suite("test_grouping_sets", "p0") { sql "drop table if exists test_query_db.test_grouping_sets_rollup" // test_grouping_select test { - sql "select k1, if(grouping(k1)=1, count(k1), 0) from test_query_db.test group by grouping sets((k1))" + sql "select /*+ SET_VAR(enable_nereids_planner=false) */ k1, if(grouping(k1)=1, count(k1), 0) from test_query_db.test group by grouping sets((k1))" exception "`k1` cannot both in select list and aggregate functions " + "when using GROUPING SETS/CUBE/ROLLUP, please use union instead." } + + test { + sql "select /*+ SET_VAR(enable_nereids_planner=true) */ k1, if(grouping(k1)=1, count(k1), 0) from test_query_db.test group by grouping sets((k1))" + exception "k1 cannot both in select list and aggregate functions " + + "when using GROUPING SETS/CUBE/ROLLUP, please use union instead." + } } From 7ee02952f10d5f68cf2b2a6a5eca960d5b98e727 Mon Sep 17 00:00:00 2001 From: starocean999 <12095047@qq.com> Date: Fri, 10 Nov 2023 15:45:08 +0800 Subject: [PATCH 2/3] fix test case --- .../grouping_sets/test_grouping_sets.groovy | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy index f2994233087925..cb78f101911c7f 100644 --- a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy +++ b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy @@ -52,14 +52,15 @@ suite("test_grouping_sets", "p0") { exception "errCode = 2, detailMessage = column: `k3` cannot both in select list and aggregate functions" } + sql """set enable_nereids_planner=true;""" test { sql """ - SELECT /*+ SET_VAR(enable_nereids_planner=true) */ k1, k2, SUM(k3) FROM test_query_db.test + SELECT k1, k2, SUM(k3) FROM test_query_db.test GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ( ), (k3) ) order by k1, k2 """ exception "errCode = 2, detailMessage = column: k3 cannot both in select list and aggregate functions" } - + sql """set enable_nereids_planner=false;""" test { sql """ SELECT /*+ SET_VAR(enable_nereids_planner=false) */ k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test @@ -68,13 +69,15 @@ suite("test_grouping_sets", "p0") { exception "errCode = 2, detailMessage = column: `k3` cannot both in select list and aggregate functions" } + sql """set enable_nereids_planner=true;""" test { sql """ - SELECT /*+ SET_VAR(enable_nereids_planner=true) */ k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test + SELECT k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test GROUP BY GROUPING SETS ((k1, k2), (k1), (k2), ( ), (k3) ) order by k1, k2 """ exception "errCode = 2, detailMessage = column: k3 cannot both in select list and aggregate functions" } + sql """set enable_nereids_planner=false;""" qt_select7 """ select k1,k2,sum(k3) from test_query_db.test where 1 = 2 group by grouping sets((k1), (k1,k2)) """ @@ -183,11 +186,12 @@ suite("test_grouping_sets", "p0") { sql "SELECT /*+ SET_VAR(enable_nereids_planner=false) */ k1 ,GROUPING(k2) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1" exception "Column `k2` in GROUP_ID() does not exist in GROUP BY clause" } - + sql """set enable_nereids_planner=true;""" test { - sql "SELECT /*+ SET_VAR(enable_nereids_planner=true) */ k1 ,GROUPING(k2) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1" + sql "SELECT k1 ,GROUPING(k2) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1" exception "Column in Grouping does not exist in GROUP BY clause" } + sql """set enable_nereids_planner=false;""" // test grouping sets id contain null data sql """drop table if exists test_query_db.test_grouping_sets_id_null""" @@ -256,9 +260,10 @@ suite("test_grouping_sets", "p0") { exception "`k1` cannot both in select list and aggregate functions " + "when using GROUPING SETS/CUBE/ROLLUP, please use union instead." } + sql """set enable_nereids_planner=true;""" test { - sql "select /*+ SET_VAR(enable_nereids_planner=true) */ k1, if(grouping(k1)=1, count(k1), 0) from test_query_db.test group by grouping sets((k1))" + sql "select k1, if(grouping(k1)=1, count(k1), 0) from test_query_db.test group by grouping sets((k1))" exception "k1 cannot both in select list and aggregate functions " + "when using GROUPING SETS/CUBE/ROLLUP, please use union instead." } From e61fca909cc12135f1cafec326cf14d633eabe27 Mon Sep 17 00:00:00 2001 From: starocean999 <12095047@qq.com> Date: Fri, 10 Nov 2023 16:55:03 +0800 Subject: [PATCH 3/3] fix case --- .../query_p0/grouping_sets/test_grouping_sets.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy index cb78f101911c7f..6564bca3509f94 100644 --- a/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy +++ b/regression-test/suites/query_p0/grouping_sets/test_grouping_sets.groovy @@ -53,6 +53,7 @@ suite("test_grouping_sets", "p0") { } sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false;""" test { sql """ SELECT k1, k2, SUM(k3) FROM test_query_db.test @@ -61,6 +62,7 @@ suite("test_grouping_sets", "p0") { exception "errCode = 2, detailMessage = column: k3 cannot both in select list and aggregate functions" } sql """set enable_nereids_planner=false;""" + sql """set enable_fallback_to_original_planner=true;""" test { sql """ SELECT /*+ SET_VAR(enable_nereids_planner=false) */ k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test @@ -70,6 +72,7 @@ suite("test_grouping_sets", "p0") { } sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false;""" test { sql """ SELECT k1, k2, SUM(k3)/(SUM(k3)+1) FROM test_query_db.test @@ -78,6 +81,7 @@ suite("test_grouping_sets", "p0") { exception "errCode = 2, detailMessage = column: k3 cannot both in select list and aggregate functions" } sql """set enable_nereids_planner=false;""" + sql """set enable_fallback_to_original_planner=true;""" qt_select7 """ select k1,k2,sum(k3) from test_query_db.test where 1 = 2 group by grouping sets((k1), (k1,k2)) """ @@ -187,11 +191,13 @@ suite("test_grouping_sets", "p0") { exception "Column `k2` in GROUP_ID() does not exist in GROUP BY clause" } sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false;""" test { sql "SELECT k1 ,GROUPING(k2) FROM test_query_db.test GROUP BY CUBE (k1) ORDER BY k1" exception "Column in Grouping does not exist in GROUP BY clause" } sql """set enable_nereids_planner=false;""" + sql """set enable_fallback_to_original_planner=true;""" // test grouping sets id contain null data sql """drop table if exists test_query_db.test_grouping_sets_id_null""" @@ -261,6 +267,7 @@ suite("test_grouping_sets", "p0") { "when using GROUPING SETS/CUBE/ROLLUP, please use union instead." } sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false;""" test { sql "select k1, if(grouping(k1)=1, count(k1), 0) from test_query_db.test group by grouping sets((k1))"