From bf7b49fdd5c66367c3f9d58bc56440ff235032ba Mon Sep 17 00:00:00 2001 From: amorynan Date: Fri, 15 Dec 2023 19:05:44 +0800 Subject: [PATCH 1/3] fix explode with array has specific precision at old planner --- .../main/java/org/apache/doris/catalog/Type.java | 1 + .../apache/doris/analysis/FunctionCallExpr.java | 16 ++++++++++++++-- .../org/apache/doris/catalog/FunctionSet.java | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index b9dc9260dd3a20..daa2dda020dfe3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -103,6 +103,7 @@ public abstract class Type { public static final ScalarType DECIMAL64 = DEFAULT_DECIMAL64; public static final ScalarType DECIMAL128 = DEFAULT_DECIMAL128; public static final ScalarType DECIMAL256 = DEFAULT_DECIMAL256; + public static final ScalarType WILDCARD_DECIMAL = ScalarType.createDecimalType(-1, -1); public static final ScalarType JSONB = new ScalarType(PrimitiveType.JSONB); // (ScalarType) ScalarType.createDecimalTypeInternal(-1, -1); public static final ScalarType DEFAULT_VARCHAR = ScalarType.createVarcharType(-1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index ff113892b6bb58..513f53028074e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1618,8 +1618,20 @@ && collectChildReturnTypes()[0].isDecimalV3()) { // now first find table function in table function sets if (isTableFnCall) { Type[] childTypes = collectChildReturnTypes(); - fn = getTableFunction(fnName.getFunction(), childTypes, - Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + if (fnName.getFunction().equalsIgnoreCase("explode") && childTypes[0].isArrayType()) { + // get origin type to match builtln func + Type[] matchFuncChildTypes = getActualArgTypes(childTypes); + fn = getTableFunction(fnName.getFunction(), matchFuncChildTypes, + Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + if (fn == null) { + throw new AnalysisException(getFunctionNotFoundError(argTypes)); + } + // set param child types + fn.setReturnType(((ArrayType) childTypes[0]).getItemType()); + } else { + fn = getTableFunction(fnName.getFunction(), childTypes, + Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF); + } if (fn == null) { throw new AnalysisException(getFunctionNotFoundError(argTypes)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index e18370560c3f7d..e5ae0d502842a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1772,6 +1772,9 @@ private void initTableFunction() { Lists.newArrayList(new ArrayType(subType)), false, "_ZN5doris19DummyTableFunctions7explodeEPN9doris_udf15FunctionContextERKNS1_13CollectionValE"); } + addTableFunctionWithCombinator(EXPLODE, Type.WILDCARD_DECIMAL, Function.NullableMode.ALWAYS_NULLABLE, + Lists.newArrayList(new ArrayType(Type.WILDCARD_DECIMAL)), false, + "_ZN5doris19DummyTableFunctions7explodeEPN9doris_udf15FunctionContextERKNS1_13CollectionValE"); } public boolean isAggFunctionName(String name) { From 06d1748231e79f2362ef3a695a0ee812eb03232d Mon Sep 17 00:00:00 2001 From: amorynan Date: Mon, 18 Dec 2023 14:06:42 +0800 Subject: [PATCH 2/3] fixed --- .../table_function/explode_array_decimal.out | 7 ++ .../explode_array_decimal.groovy | 68 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 regression-test/data/query_p0/sql_functions/table_function/explode_array_decimal.out create mode 100644 regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy diff --git a/regression-test/data/query_p0/sql_functions/table_function/explode_array_decimal.out b/regression-test/data/query_p0/sql_functions/table_function/explode_array_decimal.out new file mode 100644 index 00000000000000..3e5e665535c1a3 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/table_function/explode_array_decimal.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_old_planner -- +0.800 [0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,1,9,5,5,5,5,5,5,5,5,5,0.8,0.8,0.8,0.8,0.8] + +-- !sql_nereid -- +0.800 [0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,1,9,5,5,5,5,5,5,5,5,5,0.8,0.8,0.8,0.8,0.8] + diff --git a/regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy b/regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy new file mode 100644 index 00000000000000..60eed6b2020465 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/table_function/explode_array_decimal.groovy @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("explode_array_decimal") { + sql "DROP TABLE IF EXISTS ods_device_data_1d_inc;" + + sql """ + CREATE TABLE `ods_device_data_1d_inc` ( + `id` INT NULL, + `electricityPrice` VARCHAR(5000) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `electricityPrice`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "min_load_replica_num" = "-1", + "is_being_synced" = "false", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false", + "group_commit_interval_ms" = "10000" + );""" + + sql """insert into ods_device_data_1d_inc values(1, "[0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,0.8,1,9,5,5,5,5,5,5,5,5,5,0.8,0.8,0.8,0.8,0.8]")""" + + sql "SET enable_nereids_planner=false;" + + qt_sql_old_planner """ + SELECT * from + ( + select + e1,t.electricityPrice + from + ods_device_data_1d_inc as t + lateral view explode(cast (electricityPrice as ARRAY)) tmp1 as e1 + ) kk limit 1 + """ + + sql 'set enable_fallback_to_original_planner=false' + sql 'set enable_nereids_planner=true' + + qt_sql_nereid """ + SELECT * from + ( + select + e1,t.electricityPrice + from + ods_device_data_1d_inc as t + lateral view explode(cast (electricityPrice as ARRAY)) tmp1 as e1 + ) kk limit 1 + """ +} From 8949ca62271956395ad0277ee9573576a8be0724 Mon Sep 17 00:00:00 2001 From: amorynan Date: Tue, 19 Dec 2023 10:56:34 +0800 Subject: [PATCH 3/3] add some notes --- .../java/org/apache/doris/analysis/FunctionCallExpr.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 513f53028074e6..889554283cfedb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1618,6 +1618,11 @@ && collectChildReturnTypes()[0].isDecimalV3()) { // now first find table function in table function sets if (isTableFnCall) { Type[] childTypes = collectChildReturnTypes(); + // when we call explode> with nested decimal has specific precision and scale, + // collectChildReturnTypes will return specific precision and scale decimal type witch may not match + // builtln func we defined in fe code, because we make array_support_type is actual origin type.here we + // temp write this if to get matched explode function and then set actually decimal type from sql to + // func return type. if we switch nereid would hasn't this problems. if (fnName.getFunction().equalsIgnoreCase("explode") && childTypes[0].isArrayType()) { // get origin type to match builtln func Type[] matchFuncChildTypes = getActualArgTypes(childTypes);