From be086823b6bd0184208cb280f5f490caaf30a629 Mon Sep 17 00:00:00 2001 From: Kang Date: Mon, 27 Feb 2023 19:12:19 +0800 Subject: [PATCH 01/10] init commit for function template --- .../org/apache/doris/catalog/ArrayType.java | 21 +++ .../org/apache/doris/catalog/MapType.java | 33 +++- .../apache/doris/catalog/PrimitiveType.java | 1 + .../apache/doris/catalog/TemplateType.java | 144 ++++++++++++++++++ .../java/org/apache/doris/catalog/Type.java | 12 +- .../org/apache/doris/catalog/FunctionSet.java | 38 ++++- gensrc/script/doris_builtins_functions.py | 3 +- gensrc/script/gen_builtins_functions.py | 37 ++++- 8 files changed, 276 insertions(+), 13 deletions(-) create mode 100644 fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java index 6cc9162dc64f83..1b97ce295ff77c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -27,6 +27,7 @@ import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; +import java.util.Map; import java.util.Objects; /** @@ -92,6 +93,26 @@ public boolean matchesType(Type t) { && (((ArrayType) t).containsNull || !containsNull); } + @Override + public boolean hasTemplateType() { + return itemType.hasTemplateType(); + } + + @Override + public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + if (!(specificType instanceof ArrayType)) { + throw new TypeException(specificType + " is not ArrayType"); + } + + ArrayType o = (ArrayType) specificType; + Type newItemType = itemType; + if (itemType.hasTemplateType()) { + itemType = itemType.specializeTemplateType(o.itemType, specializedTypeMap); + } + + return new ArrayType(newItemType); + } + public static ArrayType create() { return new ArrayType(); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 6fd8da9c245d3e..7a00df8c269968 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -27,6 +27,7 @@ import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; +import java.util.Map; import java.util.Objects; /** @@ -35,16 +36,16 @@ public class MapType extends Type { @SerializedName(value = "keyType") - private final Type keyType; + private Type keyType; @SerializedName(value = "isKeyContainsNull") - private final boolean isKeyContainsNull; // Now always true + private boolean isKeyContainsNull; // Now always true @SerializedName(value = "valueType") - private final Type valueType; + private Type valueType; @SerializedName(value = "isValueContainsNull") - private final boolean isValueContainsNull; // Now always true + private boolean isValueContainsNull; // Now always true public MapType() { this.keyType = NULL; @@ -121,6 +122,30 @@ public boolean matchesType(Type t) { && (valueType.matchesType(((MapType) t).valueType)); } + @Override + public boolean hasTemplateType() { + return keyType.hasTemplateType() || valueType.hasTemplateType(); + } + + @Override + public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + if (!(specificType instanceof MapType)) { + throw new TypeException(specificType + " is not MapType"); + } + + MapType specificMapType = (MapType) specificType; + Type newKeyType = keyType; + if (keyType.hasTemplateType()) { + newKeyType = keyType.specializeTemplateType(specificMapType.keyType, specializedTypeMap); + } + Type newValueType = valueType; + if (valueType.hasTemplateType()) { + newValueType = valueType.specializeTemplateType(specificMapType.valueType, specializedTypeMap); + } + + return new MapType(newKeyType, newValueType); + } + @Override public String toString() { return toSql(0).toUpperCase(); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index c4d1a4e8d80f51..99688edd4546de 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -74,6 +74,7 @@ public enum PrimitiveType { VARIANT("VARIANT", 24, TPrimitiveType.VARIANT), // Unsupported scalar types. BINARY("BINARY", -1, TPrimitiveType.BINARY), + TEMPLATE("TEMPLATE", -1, TPrimitiveType.INVALID_TYPE), ALL("ALL", -1, TPrimitiveType.INVALID_TYPE); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java new file mode 100644 index 00000000000000..43c9fbc8ed3352 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.thrift.TColumnType; +import org.apache.doris.thrift.TTypeDesc; +import org.apache.doris.thrift.TTypeNode; +import org.apache.doris.thrift.TTypeNodeType; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.gson.annotations.SerializedName; + +import java.util.Map; +import java.util.Objects; + +/** + * Describes a TemplateType type. + */ +public class TemplateType extends Type { + + @SerializedName(value = "name") + private final String name; + + @SerializedName(value = "deducedType") + private Type deducedType; + + public TemplateType() { + this.name = null; + this.deducedType = null; + } + + public TemplateType(String name) { + this.name = name; + this.deducedType = null; + } + + @Override + public PrimitiveType getPrimitiveType() { + return PrimitiveType.TEMPLATE; + } + + public Type getDeducedType() { + return deducedType; + } + + public void setDeducedType(Type deducedType) { + this.deducedType = deducedType; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof TemplateType)) { + return false; + } + TemplateType o = (TemplateType) other; + return o.name.equals(name) + && ((o.deducedType == null && deducedType == null) + || (o.deducedType != null && deducedType != null + && o.deducedType.equals(deducedType))); + } + + @Override + public boolean matchesType(Type t) { + // matches any type + return true; + } + + @Override + public boolean hasTemplateType() { + return true; + } + + @Override + public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + if (specificType.hasTemplateType()) { + throw new TypeException(specificType + " should not hasTemplateType"); + } + + Type specializedType = specializedTypeMap.get(name); + if (specializedType != null && !specializedType.equals(specificType)) { + throw new TypeException( + String.format("can not specialize template type %s to %s since it's already specialized as %s", + name, specificType, specializedType)); + } + + if (specializedType == null) { + specializedTypeMap.put(name, specificType); + } + return specificType; + } + + @Override + public String toSql(int depth) { + return String.format("TEMPLATE<%s>", name); + } + + @Override + public String toString() { + return toSql(0).toUpperCase(); + } + + @Override + protected String prettyPrint(int lpad) { + String leftPadding = Strings.repeat(" ", lpad); + return leftPadding + toSql(); + } + + @Override + public boolean supportSubType(Type subType) { + throw new RuntimeException("supportSubType not implementd for TemplateType"); + } + + @Override + public void toThrift(TTypeDesc container) { + throw new RuntimeException("can not call toThrift on TemplateType"); + } + + @Override + public TColumnType toColumnTypeThrift() { + throw new RuntimeException("can not call toColumnTypeThrift on TemplateType"); + } + + @Override + public int hashCode() { + return Objects.hash(name, deducedType); + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index a4bd07d33a6d52..c93c14ca496ebd 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -39,6 +39,7 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -507,6 +508,14 @@ public boolean isDateV2() { return isScalarType(PrimitiveType.DATEV2); } + public boolean hasTemplateType() { + return false; + } + + public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + throw new TypeException("specializeTemplateType not implemented"); + } + /** * Returns true if Impala supports this type in the metdata. It does not mean we * can manipulate data of this type. For tables that contain columns with these @@ -1560,11 +1569,12 @@ public Integer getNumPrecRadix() { || t1 == PrimitiveType.TIMEV2 || t2 == PrimitiveType.TIMEV2 || t1 == PrimitiveType.MAP || t2 == PrimitiveType.MAP || t1 == PrimitiveType.STRUCT || t2 == PrimitiveType.STRUCT + || t1 == PrimitiveType.TEMPLATE || t2 == PrimitiveType.TEMPLATE || t1 == PrimitiveType.UNSUPPORTED || t2 == PrimitiveType.UNSUPPORTED || t1 == PrimitiveType.VARIANT || t2 == PrimitiveType.VARIANT) { continue; } - Preconditions.checkNotNull(compatibilityMatrix[i][j]); + // Preconditions.checkNotNull(compatibilityMatrix[i][j]); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 0a4bed27893a2d..1346139598296c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -27,6 +27,7 @@ import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.builtins.ScalarBuiltins; import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.common.AnalysisException; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -1226,7 +1227,7 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is // First check for identical for (Function f : fns) { if (f.compare(desc, Function.CompareMode.IS_IDENTICAL)) { - return f; + return FunctionSet.specializeTemplateFunction(f, desc); } } if (mode == Function.CompareMode.IS_IDENTICAL) { @@ -1262,6 +1263,41 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is return null; } + public static Function specializeTemplateFunction(Function templateFunction, Function requestFunction) { + try { + boolean hasTemplateType = false; + LOG.info("templateFunction signature: " + templateFunction.signatureString()); + Function specializedFunction = templateFunction; + if (templateFunction instanceof ScalarFunction) { + ScalarFunction f = (ScalarFunction) templateFunction; + specializedFunction = new ScalarFunction(f.getFunctionName(), Lists.newArrayList(f.getArgs()), + f.getReturnType(), f.hasVarArgs(), f.getSymbolName(),f.getBinaryType(), + f.isUserVisible(), f.isVectorized(), f.getNullableMode()); + } else { + // TODO + } + Type[] args = specializedFunction.getArgs(); + Map specializedTypeMap = Maps.newHashMap(); + for (int i = 0; i < args.length; i++) { + if (args[i].hasTemplateType()) { + hasTemplateType = true; + args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap); + } + } + if (specializedFunction.getReturnType().hasTemplateType()) { + hasTemplateType = true; + specializedFunction.setReturnType( + specializedFunction.getReturnType().specializeTemplateType( + requestFunction.getReturnType(), specializedTypeMap)); + } + LOG.info("specializedFunction signature: " + specializedFunction.signatureString()); + return hasTemplateType ? specializedFunction : templateFunction; + } catch (TypeException e) { + LOG.warn("specializeTemplateFunction exception", e); + return null; + } + } + /** * There are essential differences in the implementation of some functions for different * types params, which should be prohibited. diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index f650e0a28af1ed..5415b1b3fe5049 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -100,7 +100,8 @@ [['element_at', '%element_extract%'], 'STRING', ['ARRAY_STRING', 'BIGINT'], 'ALWAYS_NULLABLE'], # map element - [['element_at', '%element_extract%'], 'INT', ['MAP_STRING_INT', 'STRING'], 'ALWAYS_NULLABLE'], + # [['element_at', '%element_extract%'], 'INT', ['MAP_STRING_INT', 'STRING'], 'ALWAYS_NULLABLE'], + [['element_at', '%element_extract%'], 'V', ['MAP', 'K'], 'ALWAYS_NULLABLE', ['K', 'V']], [['arrays_overlap'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'ARRAY_BOOLEAN'], 'ALWAYS_NULLABLE'], [['arrays_overlap'], 'BOOLEAN', ['ARRAY_TINYINT', 'ARRAY_TINYINT'], 'ALWAYS_NULLABLE'], diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index 28b8812142719c..3af4708bb2d20f 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -54,6 +54,7 @@ \n\ import org.apache.doris.catalog.ArrayType;\n\ import org.apache.doris.catalog.MapType;\n\ +import org.apache.doris.catalog.TemplateType;\n\ import org.apache.doris.catalog.Type;\n\ import org.apache.doris.catalog.Function;\n\ import org.apache.doris.catalog.FunctionSet;\n\ @@ -80,7 +81,7 @@ def add_function(fn_meta_data, user_visible): """add function """ - assert len(fn_meta_data) == 4, \ + assert len(fn_meta_data) >= 4, \ "Invalid function entry in doris_builtins_functions.py:\n\t" + repr(fn_meta_data) entry = {} entry["sql_names"] = fn_meta_data[0] @@ -91,6 +92,13 @@ def add_function(fn_meta_data, user_visible): else: entry['nullable_mode'] = 'DEPEND_ON_ARGUMENT' + # process template + if len(fn_meta_data) >= 5: + entry["template_types"] = fn_meta_data[4] + print("DEBUG template fn: ", fn_meta_data, "entry: ", entry) + else: + entry["template_types"] = [] + entry["user_visible"] = user_visible meta_data_entries.append(entry) @@ -103,15 +111,32 @@ def add_function(fn_meta_data, user_visible): in[ARRAY_INT] --> out[new ArrayType(Type.INT)] in[MAP_STRING_INT] --> out[new MapType(Type.STRING,Type.INT)] """ -def generate_fe_datatype(str_type): +def generate_fe_datatype(str_type, template_types): + # process template + if str_type.strip() in template_types: + return 'new TemplateType("{}")'.format(str_type.strip()) + + template_start = str_type.find('<') + template_end = str_type.rfind('>') + if template_start >= 0 and template_end > 0: + # exclude <> + template = str_type[template_start + 1 : template_end] + print("DEBUG: template = ", template) + if str_type.startswith("ARRAY<"): + return 'new ArrayType({})'.format(generate_fe_datatype(template, template_types)) + elif str_type.startswith("MAP<"): + types = template.split(',', 2) + print("DEBUG: types = ", types) + return 'new MapType({}, {})'.format(generate_fe_datatype(types[0], template_types), generate_fe_datatype(types[1], template_types)) + if str_type.startswith("ARRAY_"): vec_type = str_type.split('_', 1); if len(vec_type) > 1 and vec_type[0] == "ARRAY": - return "new ArrayType(" + generate_fe_datatype(vec_type[1]) + ")" + return "new ArrayType(" + generate_fe_datatype(vec_type[1], template_types) + ")" if str_type.startswith("MAP_"): vec_type = str_type.split('_', 2) if len(vec_type) > 2 and vec_type[0] == "MAP": - return "new MapType(" + generate_fe_datatype(vec_type[1]) + "," + generate_fe_datatype(vec_type[2])+")" + return "new MapType(" + generate_fe_datatype(vec_type[1], template_types) + "," + generate_fe_datatype(vec_type[2], template_types)+")" if str_type == "DECIMALV2": return "Type.MAX_DECIMALV2_TYPE" if str_type == "DECIMAL32": @@ -136,7 +161,7 @@ def generate_fe_entry(entry, name): else: java_output += ", false" java_output += ", Function.NullableMode." + entry["nullable_mode"] - java_output += ", " + generate_fe_datatype(entry["ret_type"]) + java_output += ", " + generate_fe_datatype(entry["ret_type"], entry["template_types"]) # Check the last entry for varargs indicator. if entry["args"] and entry["args"][-1] == "...": @@ -145,7 +170,7 @@ def generate_fe_entry(entry, name): else: java_output += ", false" for arg in entry["args"]: - java_output += ", " + generate_fe_datatype(arg) + java_output += ", " + generate_fe_datatype(arg, entry["template_types"]) return java_output # Generates the FE builtins init file that registers all the builtins. From f9870a4569fe2e6c2d8cd25653a3401ed2190ccd Mon Sep 17 00:00:00 2001 From: Kang Date: Mon, 27 Feb 2023 23:14:18 +0800 Subject: [PATCH 02/10] first working version --- .../org/apache/doris/catalog/ArrayType.java | 5 +++-- .../org/apache/doris/catalog/MapType.java | 7 +++--- .../apache/doris/catalog/TemplateType.java | 22 ++++++++++++++----- .../java/org/apache/doris/catalog/Type.java | 3 ++- .../doris/analysis/FunctionCallExpr.java | 2 +- .../org/apache/doris/catalog/Function.java | 2 +- .../org/apache/doris/catalog/FunctionSet.java | 12 ++++++---- 7 files changed, 36 insertions(+), 17 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java index 1b97ce295ff77c..8287f974a4cb93 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -99,7 +99,8 @@ public boolean hasTemplateType() { } @Override - public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + public Type specializeTemplateType(Type specificType, Map specializedTypeMap, + boolean useSpecializedType) throws TypeException { if (!(specificType instanceof ArrayType)) { throw new TypeException(specificType + " is not ArrayType"); } @@ -107,7 +108,7 @@ public Type specializeTemplateType(Type specificType, Map speciali ArrayType o = (ArrayType) specificType; Type newItemType = itemType; if (itemType.hasTemplateType()) { - itemType = itemType.specializeTemplateType(o.itemType, specializedTypeMap); + itemType = itemType.specializeTemplateType(o.itemType, specializedTypeMap, useSpecializedType); } return new ArrayType(newItemType); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 7a00df8c269968..720043dc845b1b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -128,7 +128,8 @@ public boolean hasTemplateType() { } @Override - public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + public Type specializeTemplateType(Type specificType, Map specializedTypeMap, + boolean useSpecializedType) throws TypeException { if (!(specificType instanceof MapType)) { throw new TypeException(specificType + " is not MapType"); } @@ -136,11 +137,11 @@ public Type specializeTemplateType(Type specificType, Map speciali MapType specificMapType = (MapType) specificType; Type newKeyType = keyType; if (keyType.hasTemplateType()) { - newKeyType = keyType.specializeTemplateType(specificMapType.keyType, specializedTypeMap); + newKeyType = keyType.specializeTemplateType(specificMapType.keyType, specializedTypeMap, useSpecializedType); } Type newValueType = valueType; if (valueType.hasTemplateType()) { - newValueType = valueType.specializeTemplateType(specificMapType.valueType, specializedTypeMap); + newValueType = valueType.specializeTemplateType(specificMapType.valueType, specializedTypeMap, useSpecializedType); } return new MapType(newKeyType, newValueType); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index 43c9fbc8ed3352..f8bdb75a2ff692 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -88,13 +88,25 @@ public boolean hasTemplateType() { } @Override - public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + public Type specializeTemplateType(Type specificType, Map specializedTypeMap, + boolean useSpecializedType) throws TypeException { if (specificType.hasTemplateType()) { throw new TypeException(specificType + " should not hasTemplateType"); } Type specializedType = specializedTypeMap.get(name); - if (specializedType != null && !specializedType.equals(specificType)) { + if (useSpecializedType) { + if (specializedType == null) { + throw new TypeException("template type " + name + " is not specialized yet"); + } + return specializedType; + } + + if (specializedType != null + && !specificType.equals(specializedType) + && !specificType.matchesType(specializedType) + && !Type.isImplicitlyCastable(specificType, specializedType, true) + && !Type.canCastTo(specificType, specializedType)) { throw new TypeException( String.format("can not specialize template type %s to %s since it's already specialized as %s", name, specificType, specializedType)); @@ -103,17 +115,17 @@ public Type specializeTemplateType(Type specificType, Map speciali if (specializedType == null) { specializedTypeMap.put(name, specificType); } - return specificType; + return specializedTypeMap.get(name); } @Override public String toSql(int depth) { - return String.format("TEMPLATE<%s>", name); + return name; } @Override public String toString() { - return toSql(0).toUpperCase(); + return toSql(0).toUpperCase(); } @Override diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index c93c14ca496ebd..c4b6ca0e004968 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -512,7 +512,8 @@ public boolean hasTemplateType() { return false; } - public Type specializeTemplateType(Type specificType, Map specializedTypeMap) throws TypeException { + public Type specializeTemplateType(Type specificType, Map specializedTypeMap, + boolean useSpecializedType) throws TypeException { throw new TypeException("specializeTemplateType not implemented"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 9e101275a046c5..dc556832d673b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1369,7 +1369,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { } } - if (!fn.getFunctionName().getFunction().equals(ELEMENT_EXTRACT_FN_NAME)) { + if (!fn.getFunctionName().getFunction().equals("TODO")) { Type[] args = fn.getArgs(); if (args.length > 0) { // Implicitly cast all the children to match the function if necessary diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index f1a608a9b83f92..fc8faa5536e312 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -376,7 +376,7 @@ private boolean isIdentical(Function o) { return false; } for (int i = 0; i < this.argTypes.length; ++i) { - if (!o.argTypes[i].matchesType(this.argTypes[i])) { + if (!this.argTypes[i].matchesType(o.argTypes[i])) { return false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 1346139598296c..680a62ad05b3dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1266,7 +1266,10 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is public static Function specializeTemplateFunction(Function templateFunction, Function requestFunction) { try { boolean hasTemplateType = false; - LOG.info("templateFunction signature: " + templateFunction.signatureString()); + LOG.info("templateFunction signature: " + templateFunction.signatureString() + + " return: " + templateFunction.getReturnType()); + LOG.info("requestFunction signature: " + requestFunction.signatureString() + + " return: " + requestFunction.getReturnType()); Function specializedFunction = templateFunction; if (templateFunction instanceof ScalarFunction) { ScalarFunction f = (ScalarFunction) templateFunction; @@ -1281,16 +1284,17 @@ public static Function specializeTemplateFunction(Function templateFunction, Fun for (int i = 0; i < args.length; i++) { if (args[i].hasTemplateType()) { hasTemplateType = true; - args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap); + args[i] = args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap, false); } } if (specializedFunction.getReturnType().hasTemplateType()) { hasTemplateType = true; specializedFunction.setReturnType( specializedFunction.getReturnType().specializeTemplateType( - requestFunction.getReturnType(), specializedTypeMap)); + requestFunction.getReturnType(), specializedTypeMap, true)); } - LOG.info("specializedFunction signature: " + specializedFunction.signatureString()); + LOG.info("specializedFunction signature: " + specializedFunction.signatureString() + + " return: " + specializedFunction.getReturnType()); return hasTemplateType ? specializedFunction : templateFunction; } catch (TypeException e) { LOG.warn("specializeTemplateFunction exception", e); From 56e6496a7ab7852c34757ad6e393da48a28a7686 Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 2 Mar 2023 15:01:36 +0800 Subject: [PATCH 03/10] fix whitespace problem in template function --- gensrc/script/gen_builtins_functions.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index 3af4708bb2d20f..5776be59c61bb5 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -112,10 +112,14 @@ def add_function(fn_meta_data, user_visible): in[MAP_STRING_INT] --> out[new MapType(Type.STRING,Type.INT)] """ def generate_fe_datatype(str_type, template_types): + # delete whitespace + str_type = str_type.replace(' ', '').replace('\t', '') + # process template - if str_type.strip() in template_types: - return 'new TemplateType("{}")'.format(str_type.strip()) + if str_type in template_types: + return 'new TemplateType("{}")'.format(str_type) + # process Array, Map, Struct template template_start = str_type.find('<') template_end = str_type.rfind('>') if template_start >= 0 and template_end > 0: @@ -129,6 +133,7 @@ def generate_fe_datatype(str_type, template_types): print("DEBUG: types = ", types) return 'new MapType({}, {})'.format(generate_fe_datatype(types[0], template_types), generate_fe_datatype(types[1], template_types)) + # lagacy Array, Map syntax if str_type.startswith("ARRAY_"): vec_type = str_type.split('_', 1); if len(vec_type) > 1 and vec_type[0] == "ARRAY": From 5621e96533890d88f679caf3112f9523788d5752 Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 2 Mar 2023 15:02:16 +0800 Subject: [PATCH 04/10] make tempalte function specialize and match more robust --- .../apache/doris/catalog/TemplateType.java | 4 +- .../doris/analysis/FunctionCallExpr.java | 59 +++++++++---------- .../org/apache/doris/catalog/Function.java | 12 +++- .../org/apache/doris/catalog/FunctionSet.java | 32 +++++++++- 4 files changed, 71 insertions(+), 36 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index f8bdb75a2ff692..18dfeec5a8bbcc 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -78,8 +78,8 @@ public boolean equals(Object other) { @Override public boolean matchesType(Type t) { - // matches any type - return true; + // not matches any type + return false; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index dc556832d673b2..51b6b75cad3f73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1369,37 +1369,34 @@ && collectChildReturnTypes()[0].isDecimalV3()) { } } - if (!fn.getFunctionName().getFunction().equals("TODO")) { - Type[] args = fn.getArgs(); - if (args.length > 0) { - // Implicitly cast all the children to match the function if necessary - for (int i = 0; i < argTypes.length - orderByElements.size(); ++i) { - // For varargs, we must compare with the last type in callArgs.argTypes. - int ix = Math.min(args.length - 1, i); - if (fnName.getFunction().equalsIgnoreCase("money_format") - && children.get(0).getType().isDecimalV3() && args[ix].isDecimalV3()) { - continue; - } else if (fnName.getFunction().equalsIgnoreCase("array") - && (children.get(0).getType().isDecimalV3() && args[ix].isDecimalV3() - || children.get(0).getType().isDatetimeV2() && args[ix].isDatetimeV2())) { - continue; - } else if ((fnName.getFunction().equalsIgnoreCase("array_min") || fnName.getFunction() - .equalsIgnoreCase("array_max") || fnName.getFunction().equalsIgnoreCase("element_at")) - && (( - children.get(0).getType().isDecimalV3() && ((ArrayType) args[ix]).getItemType() - .isDecimalV3()) - || (children.get(0).getType().isDatetimeV2() - && ((ArrayType) args[ix]).getItemType().isDatetimeV2()) - || (children.get(0).getType().isDecimalV2() - && ((ArrayType) args[ix]).getItemType().isDecimalV2()))) { - continue; - } else if (!argTypes[i].matchesType(args[ix]) - && !(argTypes[i].isDateOrDateTime() && args[ix].isDateOrDateTime()) - && (!fn.getReturnType().isDecimalV3() - || (argTypes[i].isValid() && !argTypes[i].isDecimalV3() - && args[ix].isDecimalV3()))) { - uncheckedCastChild(args[ix], i); - } + Type[] args = fn.getArgs(); + if (args.length > 0) { + // Implicitly cast all the children to match the function if necessary + for (int i = 0; i < argTypes.length - orderByElements.size(); ++i) { + // For varargs, we must compare with the last type in callArgs.argTypes. + int ix = Math.min(args.length - 1, i); + if (fnName.getFunction().equalsIgnoreCase("money_format") + && children.get(0).getType().isDecimalV3() && args[ix].isDecimalV3()) { + continue; + } else if (fnName.getFunction().equalsIgnoreCase("array") + && (children.get(0).getType().isDecimalV3() && args[ix].isDecimalV3() + || children.get(0).getType().isDatetimeV2() && args[ix].isDatetimeV2())) { + continue; + } else if ((fnName.getFunction().equalsIgnoreCase("array_min") || fnName.getFunction() + .equalsIgnoreCase("array_max") || fnName.getFunction().equalsIgnoreCase("element_at")) + && (( + children.get(0).getType().isDecimalV3() && ((ArrayType) args[ix]).getItemType() + .isDecimalV3()) + || (children.get(0).getType().isDatetimeV2() + && ((ArrayType) args[ix]).getItemType().isDatetimeV2()) + || (children.get(0).getType().isDecimalV2() + && ((ArrayType) args[ix]).getItemType().isDecimalV2()))) { + continue; + } else if (!argTypes[i].matchesType(args[ix]) && !( + argTypes[i].isDateOrDateTime() && args[ix].isDateOrDateTime()) + && (!fn.getReturnType().isDecimalV3() + || (argTypes[i].isValid() && !argTypes[i].isDecimalV3() && args[ix].isDecimalV3()))) { + uncheckedCastChild(args[ix], i); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index fc8faa5536e312..7fa34865cc4189 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -376,7 +376,7 @@ private boolean isIdentical(Function o) { return false; } for (int i = 0; i < this.argTypes.length; ++i) { - if (!this.argTypes[i].matchesType(o.argTypes[i])) { + if (!o.argTypes[i].matchesType(this.argTypes[i])) { return false; } } @@ -814,4 +814,14 @@ public void checkWritable() throws UserException { throw new UserException("failed to serialize function: " + functionName(), t); } } + + public boolean hasTemplateType() { + for (Type t : getArgs()) { + if (t.hasTemplateType()) { + return true; + } + } + + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 680a62ad05b3dc..f022d823e1ab7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1224,10 +1224,38 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is return null; } + List normalFunctions = Lists.newArrayList(); + List templateFunctions = Lists.newArrayList(); + for (Function fn : fns) { + if (fn.hasTemplateType()) { + templateFunctions.add(fn); + } else { + normalFunctions.add(fn); + } + } + + // try normal functions first + Function fn = getFunction(desc, mode, normalFunctions); + if (fn != null) { + return fn; + } + + // then specialize template functions and try them + List specializedTemplateFunctions = Lists.newArrayList(); + for (Function f : templateFunctions) { + f = FunctionSet.specializeTemplateFunction(f, desc); + if (f != null) { + specializedTemplateFunctions.add(f); + } + } + return getFunction(desc, mode, specializedTemplateFunctions); + } + + private Function getFunction(Function desc, Function.CompareMode mode, List fns) { // First check for identical for (Function f : fns) { if (f.compare(desc, Function.CompareMode.IS_IDENTICAL)) { - return FunctionSet.specializeTemplateFunction(f, desc); + return f; } } if (mode == Function.CompareMode.IS_IDENTICAL) { @@ -1299,7 +1327,7 @@ public static Function specializeTemplateFunction(Function templateFunction, Fun } catch (TypeException e) { LOG.warn("specializeTemplateFunction exception", e); return null; - } + } } /** From 47b521eb997e243386e7285009098ab6d8daa527 Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 2 Mar 2023 15:50:20 +0800 Subject: [PATCH 05/10] delete useless code --- .../org/apache/doris/catalog/MapType.java | 8 +++---- .../apache/doris/catalog/PrimitiveType.java | 2 +- .../apache/doris/catalog/TemplateType.java | 24 ++----------------- .../java/org/apache/doris/catalog/Type.java | 10 ++++++-- .../org/apache/doris/catalog/Function.java | 2 +- .../org/apache/doris/catalog/FunctionSet.java | 10 ++++---- gensrc/script/doris_builtins_functions.py | 1 - gensrc/script/gen_builtins_functions.py | 3 --- 8 files changed, 21 insertions(+), 39 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 720043dc845b1b..386a4cf740b390 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -36,16 +36,16 @@ public class MapType extends Type { @SerializedName(value = "keyType") - private Type keyType; + private final Type keyType; @SerializedName(value = "isKeyContainsNull") - private boolean isKeyContainsNull; // Now always true + private final boolean isKeyContainsNull; // Now always true @SerializedName(value = "valueType") - private Type valueType; + private final Type valueType; @SerializedName(value = "isValueContainsNull") - private boolean isValueContainsNull; // Now always true + private final boolean isValueContainsNull; // Now always true public MapType() { this.keyType = NULL; diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index 99688edd4546de..fdcbfaedef5253 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -72,9 +72,9 @@ public enum PrimitiveType { STRUCT("STRUCT", 16, TPrimitiveType.STRUCT), STRING("STRING", 16, TPrimitiveType.STRING), VARIANT("VARIANT", 24, TPrimitiveType.VARIANT), + TEMPLATE("TEMPLATE", -1, TPrimitiveType.INVALID_TYPE), // Unsupported scalar types. BINARY("BINARY", -1, TPrimitiveType.BINARY), - TEMPLATE("TEMPLATE", -1, TPrimitiveType.INVALID_TYPE), ALL("ALL", -1, TPrimitiveType.INVALID_TYPE); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index 18dfeec5a8bbcc..90464022e4de1c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -38,17 +38,8 @@ public class TemplateType extends Type { @SerializedName(value = "name") private final String name; - @SerializedName(value = "deducedType") - private Type deducedType; - - public TemplateType() { - this.name = null; - this.deducedType = null; - } - public TemplateType(String name) { this.name = name; - this.deducedType = null; } @Override @@ -56,24 +47,13 @@ public PrimitiveType getPrimitiveType() { return PrimitiveType.TEMPLATE; } - public Type getDeducedType() { - return deducedType; - } - - public void setDeducedType(Type deducedType) { - this.deducedType = deducedType; - } - @Override public boolean equals(Object other) { if (!(other instanceof TemplateType)) { return false; } TemplateType o = (TemplateType) other; - return o.name.equals(name) - && ((o.deducedType == null && deducedType == null) - || (o.deducedType != null && deducedType != null - && o.deducedType.equals(deducedType))); + return o.name.equals(name); } @Override @@ -151,6 +131,6 @@ public TColumnType toColumnTypeThrift() { @Override public int hashCode() { - return Objects.hash(name, deducedType); + return name.hashCode(); } } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index c4b6ca0e004968..4f7f5cb96e3f3a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -512,9 +512,15 @@ public boolean hasTemplateType() { return false; } + // return a new type without template type, by specialize tempalte type in this type public Type specializeTemplateType(Type specificType, Map specializedTypeMap, boolean useSpecializedType) throws TypeException { - throw new TypeException("specializeTemplateType not implemented"); + if (hasTemplateType()) { + // throw exception by default, sub class should specialize tempalte type properly + throw new TypeException("specializeTemplateType not implemented"); + } else { + return this; + } } /** @@ -1575,7 +1581,7 @@ public Integer getNumPrecRadix() { || t1 == PrimitiveType.VARIANT || t2 == PrimitiveType.VARIANT) { continue; } - // Preconditions.checkNotNull(compatibilityMatrix[i][j]); + Preconditions.checkNotNull(compatibilityMatrix[i][j]); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 7fa34865cc4189..8997b1bf3204a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -815,7 +815,7 @@ public void checkWritable() throws UserException { } } - public boolean hasTemplateType() { + public boolean hasTemplateArg() { for (Type t : getArgs()) { if (t.hasTemplateType()) { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index f022d823e1ab7d..f429d50f4c3941 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1227,7 +1227,7 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is List normalFunctions = Lists.newArrayList(); List templateFunctions = Lists.newArrayList(); for (Function fn : fns) { - if (fn.hasTemplateType()) { + if (fn.hasTemplateArg()) { templateFunctions.add(fn); } else { normalFunctions.add(fn); @@ -1294,9 +1294,9 @@ private Function getFunction(Function desc, Function.CompareMode mode, List specializedTypeMap = Maps.newHashMap(); @@ -1321,7 +1321,7 @@ public static Function specializeTemplateFunction(Function templateFunction, Fun specializedFunction.getReturnType().specializeTemplateType( requestFunction.getReturnType(), specializedTypeMap, true)); } - LOG.info("specializedFunction signature: " + specializedFunction.signatureString() + LOG.debug("specializedFunction signature: " + specializedFunction.signatureString() + " return: " + specializedFunction.getReturnType()); return hasTemplateType ? specializedFunction : templateFunction; } catch (TypeException e) { diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 5415b1b3fe5049..b10e8500cf4e7f 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -100,7 +100,6 @@ [['element_at', '%element_extract%'], 'STRING', ['ARRAY_STRING', 'BIGINT'], 'ALWAYS_NULLABLE'], # map element - # [['element_at', '%element_extract%'], 'INT', ['MAP_STRING_INT', 'STRING'], 'ALWAYS_NULLABLE'], [['element_at', '%element_extract%'], 'V', ['MAP', 'K'], 'ALWAYS_NULLABLE', ['K', 'V']], [['arrays_overlap'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'ARRAY_BOOLEAN'], 'ALWAYS_NULLABLE'], diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index 5776be59c61bb5..7858010b82c947 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -95,7 +95,6 @@ def add_function(fn_meta_data, user_visible): # process template if len(fn_meta_data) >= 5: entry["template_types"] = fn_meta_data[4] - print("DEBUG template fn: ", fn_meta_data, "entry: ", entry) else: entry["template_types"] = [] @@ -125,12 +124,10 @@ def generate_fe_datatype(str_type, template_types): if template_start >= 0 and template_end > 0: # exclude <> template = str_type[template_start + 1 : template_end] - print("DEBUG: template = ", template) if str_type.startswith("ARRAY<"): return 'new ArrayType({})'.format(generate_fe_datatype(template, template_types)) elif str_type.startswith("MAP<"): types = template.split(',', 2) - print("DEBUG: types = ", types) return 'new MapType({}, {})'.format(generate_fe_datatype(types[0], template_types), generate_fe_datatype(types[1], template_types)) # lagacy Array, Map syntax From 2a4be8d2648676df1d26f8089ccd5498c6a1d1c2 Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 2 Mar 2023 16:16:38 +0800 Subject: [PATCH 06/10] fix checkstyle --- .../src/main/java/org/apache/doris/catalog/MapType.java | 6 ++++-- .../main/java/org/apache/doris/catalog/TemplateType.java | 5 ----- .../main/java/org/apache/doris/catalog/FunctionSet.java | 7 +++---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 386a4cf740b390..735b25a0923e7b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -137,11 +137,13 @@ public Type specializeTemplateType(Type specificType, Map speciali MapType specificMapType = (MapType) specificType; Type newKeyType = keyType; if (keyType.hasTemplateType()) { - newKeyType = keyType.specializeTemplateType(specificMapType.keyType, specializedTypeMap, useSpecializedType); + newKeyType = keyType.specializeTemplateType( + specificMapType.keyType, specializedTypeMap, useSpecializedType); } Type newValueType = valueType; if (valueType.hasTemplateType()) { - newValueType = valueType.specializeTemplateType(specificMapType.valueType, specializedTypeMap, useSpecializedType); + newValueType = valueType.specializeTemplateType( + specificMapType.valueType, specializedTypeMap, useSpecializedType); } return new MapType(newKeyType, newValueType); diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index 90464022e4de1c..a9455752fa1475 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -19,16 +19,11 @@ import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TTypeDesc; -import org.apache.doris.thrift.TTypeNode; -import org.apache.doris.thrift.TTypeNodeType; -import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; import java.util.Map; -import java.util.Objects; /** * Describes a TemplateType type. diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index f429d50f4c3941..cb501817b4a484 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -27,7 +27,6 @@ import org.apache.doris.analysis.MatchPredicate; import org.apache.doris.builtins.ScalarBuiltins; import org.apache.doris.catalog.Function.NullableMode; -import org.apache.doris.common.AnalysisException; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -1302,7 +1301,7 @@ public static Function specializeTemplateFunction(Function templateFunction, Fun if (templateFunction instanceof ScalarFunction) { ScalarFunction f = (ScalarFunction) templateFunction; specializedFunction = new ScalarFunction(f.getFunctionName(), Lists.newArrayList(f.getArgs()), - f.getReturnType(), f.hasVarArgs(), f.getSymbolName(),f.getBinaryType(), + f.getReturnType(), f.hasVarArgs(), f.getSymbolName(), f.getBinaryType(), f.isUserVisible(), f.isVectorized(), f.getNullableMode()); } else { // TODO(xk) @@ -1318,8 +1317,8 @@ public static Function specializeTemplateFunction(Function templateFunction, Fun if (specializedFunction.getReturnType().hasTemplateType()) { hasTemplateType = true; specializedFunction.setReturnType( - specializedFunction.getReturnType().specializeTemplateType( - requestFunction.getReturnType(), specializedTypeMap, true)); + specializedFunction.getReturnType().specializeTemplateType( + requestFunction.getReturnType(), specializedTypeMap, true)); } LOG.debug("specializedFunction signature: " + specializedFunction.signatureString() + " return: " + specializedFunction.getReturnType()); From 85422d580ee150c46c82fc8c6bce52a6e6b2365d Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 2 Mar 2023 17:16:35 +0800 Subject: [PATCH 07/10] add map[key] testcase for multi map kv types --- .../test_map_load_and_function.out | 53 ++++++++++++------- .../test_map_load_and_function.groovy | 33 ++++++++++-- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out index 1b7eb1f5a3d7cb..8e3b7905a4cc0a 100644 --- a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out +++ b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out @@ -1,5 +1,5 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !select -- +-- !select_all -- 1 \N 2 {' 11amory ':23, 'beat':20, ' clever ':66} 3 {'k1':31, 'k2':300} @@ -16,22 +16,37 @@ 15 {'':2, 'k2':0} 16 {null:null} --- !select -- -\N -\N -300 -\N -\N -400 -\N -\N -\N -\N -\N -\N -\N -\N -130 -0 -\N +-- !select_m -- +1 \N +2 \N +3 300 +4 \N +5 \N +6 400 +7 \N +8 \N +9 \N +10 \N +11 \N +12 \N +13 \N +15 0 +16 \N +17 \N +18 130 + +-- !select_m1 -- +1 100 200 \N + +-- !select_m2 -- +1 k1 k2 \N + +-- !select_m3 -- +1 v1 v2 \N + +-- !select_m4 -- +1 10000 20000 \N + +-- !select_m5 -- +1 100 200 \N diff --git a/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy index d796c08b2eaa8e..38e4d295ca1a5a 100644 --- a/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy +++ b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy @@ -63,12 +63,37 @@ suite("test_map_load_and_function", "p0") { } // check result - qt_select "SELECT * FROM ${testTable} ORDER BY id" + qt_select_all "SELECT * FROM ${testTable} ORDER BY id" // insert into valid json rows - sql """INSERT INTO ${testTable} VALUES(12, NULL)""" - sql """INSERT INTO ${testTable} VALUES(13, {"k1":100, "k2": 130})""" + sql """INSERT INTO ${testTable} VALUES(17, NULL)""" + sql """INSERT INTO ${testTable} VALUES(18, {"k1":100, "k2": 130})""" // map element_at - qt_select "SELECT m['k2'] FROM ${testTable} ORDER BY id" + qt_select_m "SELECT id, m['k2'] FROM ${testTable} ORDER BY id" + + + testTable = "tbl_test_map2" + sql "DROP TABLE IF EXISTS ${testTable}" + sql """ + CREATE TABLE IF NOT EXISTS ${testTable} ( + id INT, + `m1` MAP NULL, + `m2` MAP NULL, + `m3` MAP NULL, + `m4` MAP NULL, + `m5` MAP NULL + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + sql """INSERT INTO ${testTable} VALUES(1, {'k1':100, 'k2':200}, {100:'k1', 200:'k2'}, {'k1':'v1', 'k2':'v2'}, {100:10000, 200:20000}, {10000:100, 20000:200})""" + + // map element_at + qt_select_m1 "SELECT id, m1['k1'], m1['k2'], m1['nokey'] FROM ${testTable} ORDER BY id" + qt_select_m2 "SELECT id, m2[100], m2[200], m1[300] FROM ${testTable} ORDER BY id" + qt_select_m3 "SELECT id, m3['k1'], m3['k2'], m3['nokey'] FROM ${testTable} ORDER BY id" + qt_select_m4 "SELECT id, m4[100], m4[200], m4[300] FROM ${testTable} ORDER BY id" + qt_select_m5 "SELECT id, m5[10000], m5[20000], m5[30000] FROM ${testTable} ORDER BY id" } From 0153e15d74a4aea05bb7facbfa3524d4d1158089 Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 2 Mar 2023 17:44:22 +0800 Subject: [PATCH 08/10] fix ArrayType specializeTemplateType --- .../src/main/java/org/apache/doris/catalog/ArrayType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java index 8287f974a4cb93..a28f3c030236b1 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java @@ -108,7 +108,7 @@ public Type specializeTemplateType(Type specificType, Map speciali ArrayType o = (ArrayType) specificType; Type newItemType = itemType; if (itemType.hasTemplateType()) { - itemType = itemType.specializeTemplateType(o.itemType, specializedTypeMap, useSpecializedType); + newItemType = itemType.specializeTemplateType(o.itemType, specializedTypeMap, useSpecializedType); } return new ArrayType(newItemType); From bc4d8c0373dceed31554de91e562a4210a8c2978 Mon Sep 17 00:00:00 2001 From: Kang Date: Thu, 2 Mar 2023 17:51:09 +0800 Subject: [PATCH 09/10] add doc in builtins_functions.py --- gensrc/script/doris_builtins_functions.py | 6 +++++- gensrc/script/gen_builtins_functions.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index b10e8500cf4e7f..b891b48513dda5 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -25,7 +25,7 @@ # It contains all the meta data that describes the function. # The format is: -# [sql aliases], , [], +# [sql aliases], , [], , [template_types] # # 'sql aliases' are the function names that can be used from sql. There must be at least # one per function. @@ -33,6 +33,10 @@ # 'nullable mode' reflects whether the return value of the function is null. See @Function.NullableMode # for the specific mode and meaning. # +# 'template_types' is for template function just like C++. It is optional list. +# eg. [['element_at', '%element_extract%'], 'V', ['MAP', 'K'], 'ALWAYS_NULLABLE', ['K', 'V']], +# 'K' and 'V' is type template and will be specialized at runtime in FE to match specific args. +# visible_functions = [ # Bit and Byte functions # For functions corresponding to builtin operators, we can reuse the implementations diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index 7858010b82c947..2aa6c74fcd96e9 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -74,7 +74,7 @@ # This contains all the metadata to describe all the builtins. # Each meta data entry is itself a map to store all the meta data -# - fn_name, ret_type, args, symbol, sql_names +# - fn_name, ret_type, args, symbol, sql_names, template_types(optional) meta_data_entries = [] # Read in the function and add it to the meta_data_entries map From 4634de0492ffee1cfd7897ecf809967a301a8cff Mon Sep 17 00:00:00 2001 From: Kang Date: Fri, 3 Mar 2023 21:59:02 +0800 Subject: [PATCH 10/10] fix comment --- .../src/main/java/org/apache/doris/catalog/MapType.java | 7 ++++++- .../main/java/org/apache/doris/catalog/TemplateType.java | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 735b25a0923e7b..e72df777100691 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -146,7 +146,12 @@ public Type specializeTemplateType(Type specificType, Map speciali specificMapType.valueType, specializedTypeMap, useSpecializedType); } - return new MapType(newKeyType, newValueType); + Type newMapType = new MapType(newKeyType, newValueType); + if (Type.canCastTo(specificType, newMapType)) { + return newMapType; + } else { + throw new TypeException(specificType + " can not cast to specialize type " + newMapType); + } } @Override diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index a9455752fa1475..58f3fbe952c451 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -26,7 +26,8 @@ import java.util.Map; /** - * Describes a TemplateType type. + * Describes a TemplateType type, used for SQL function argument and return type, + * NOT used for table column type. */ public class TemplateType extends Type { @@ -65,7 +66,7 @@ public boolean hasTemplateType() { @Override public Type specializeTemplateType(Type specificType, Map specializedTypeMap, boolean useSpecializedType) throws TypeException { - if (specificType.hasTemplateType()) { + if (specificType.hasTemplateType() && !specificType.isNull()) { throw new TypeException(specificType + " should not hasTemplateType"); }