From 679564f84231e9781d819bb4862b47d70a61a870 Mon Sep 17 00:00:00 2001 From: xy720 Date: Mon, 20 Mar 2023 22:21:29 +0800 Subject: [PATCH 1/5] save --- .../org/apache/doris/catalog/StructType.java | 97 +++++++++++++++++++ .../apache/doris/catalog/TemplateType.java | 46 ++++++++- .../java/org/apache/doris/catalog/Type.java | 23 ++++- .../org/apache/doris/catalog/Function.java | 43 ++++++++ .../org/apache/doris/catalog/FunctionSet.java | 36 +++++-- gensrc/script/doris_builtins_functions.py | 4 + gensrc/script/gen_builtins_functions.py | 2 + 7 files changed, 242 insertions(+), 9 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java index 2061f08a8c58e0..1d9f1c47ce1f13 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java @@ -32,7 +32,10 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; /** * Describes a STRUCT type. STRUCT types have a list of named struct fields. @@ -58,6 +61,15 @@ public StructType(ArrayList fields) { } } + public StructType(List types) { + Preconditions.checkNotNull(types); + ArrayList newFields = new ArrayList<>(); + for (Type type : types) { + newFields.add(new StructField(type)); + } + this.fields = newFields; + } + public StructType() { this.fields = Lists.newArrayList(); } @@ -171,6 +183,91 @@ public boolean matchesType(Type t) { return true; } + @Override + public boolean hasTemplateType() { + for (StructField field : fields) { + if (field.type.hasTemplateType()) { + return true; + } + } + return false; + } + + @Override + public Type specializeTemplateType(Type specificType, Map specializedTypeMap, + boolean useSpecializedType) throws TypeException { + StructType specificStructType = null; + if (specificType instanceof StructType) { + specificStructType = (StructType) specificType; + } else if (!useSpecializedType) { + throw new TypeException(specificType + " is not StructType"); + } + + List newTypes = Lists.newArrayList(); + for (int i = 0; i < fields.size(); i++) { + if (fields.get(i).type.hasTemplateType()) { + newTypes.add(fields.get(i).type.specializeTemplateType( + specificStructType != null ? specificStructType.fields.get(i).type : specificType, + specializedTypeMap, useSpecializedType)); + } + } + + Type newStructType = new StructType(newTypes); + if (Type.canCastTo(specificType, newStructType) + || (useSpecializedType && !(specificType instanceof StructType))) { + return newStructType; + } else { + throw new TypeException(specificType + " can not cast to specialize type " + newStructType); + } + } + + @Override + public boolean needExpandTemplateType() { + Preconditions.checkNotNull(fields); + return fields.get(fields.size() - 1).type.needExpandTemplateType(); + } + + /** + * A struct variadic template is like `STRUCT` or `STRUCT`... + * So that we only need to expand the last field in struct variadic template. + */ + @Override + public void collectTemplateExpandSize(Type[] args, Map expandSizeMap) throws TypeException { + Preconditions.checkState(needExpandTemplateType()); + if (args == null || args.length == 0) { + throw new TypeException("can not expand template type in struct since input args is empty."); + } + if (!(args[0] instanceof StructType)) { + throw new TypeException(args[0] + " is not StructType"); + } + StructType structType = (StructType) args[0]; + if (structType.fields.size() < fields.size()) { + throw new TypeException("the field size of input struct type " + structType + + " is less than struct template " + this); + } + Type[] types = structType.fields.subList(fields.size() - 1, structType.fields.size()) + .stream().map(field -> field.type).toArray(Type[]::new); + // only the last field is expandable + fields.get(fields.size() - 1).type.collectTemplateExpandSize(types, expandSizeMap); + } + + /** + * A struct variadic template is like `STRUCT` or `STRUCT`... + * This method is used to expand a variadic template. + * e.g. Expand `STRUCT` to `STRUCT` + */ + @Override + public List expandVariadicTemplateType(Map expandSizeMap) { + Type type = fields.get(fields.size() - 1).type; + if (type.needExpandTemplateType()) { + List types = fields.subList(0, fields.size() - 1).stream().map(field -> field.type) + .collect(Collectors.toList()); + types.addAll(type.expandVariadicTemplateType(expandSizeMap)); + return Lists.newArrayList(new StructType(types)); + } + return Lists.newArrayList(this); + } + @Override public boolean equals(Object other) { if (!(other instanceof StructType)) { diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index 58f3fbe952c451..8c90834cde8fa3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -20,9 +20,13 @@ import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TTypeDesc; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; +import java.util.Arrays; +import java.util.List; import java.util.Map; /** @@ -34,8 +38,16 @@ public class TemplateType extends Type { @SerializedName(value = "name") private final String name; - public TemplateType(String name) { + @SerializedName(value = "isVariadic") + private final boolean isVariadic; + + public TemplateType(String name, boolean isVariadic) { this.name = name; + this.isVariadic = isVariadic; + } + + public TemplateType(String name) { + this(name, false); } @Override @@ -49,7 +61,7 @@ public boolean equals(Object other) { return false; } TemplateType o = (TemplateType) other; - return o.name.equals(name); + return o.name.equals(name) && o.isVariadic == isVariadic; } @Override @@ -63,6 +75,11 @@ public boolean hasTemplateType() { return true; } + @Override + public boolean needExpandTemplateType() { + return isVariadic; + } + @Override public Type specializeTemplateType(Type specificType, Map specializedTypeMap, boolean useSpecializedType) throws TypeException { @@ -94,6 +111,31 @@ public Type specializeTemplateType(Type specificType, Map speciali return specializedTypeMap.get(name); } + @Override + public void collectTemplateExpandSize(Type[] args, Map expandSizeMap) + throws TypeException { + Preconditions.checkState(isVariadic); + expandSizeMap.computeIfAbsent(name, k -> args.length); + if (expandSizeMap.get(name) != args.length) { + throw new TypeException( + String.format("can not expand variadic template type %s to %s size since it's " + + "already expand as %s size", name, args.length, expandSizeMap.get(name))); + } + } + + @Override + public List expandVariadicTemplateType(Map expandSizeMap) { + if (needExpandTemplateType() && expandSizeMap.containsKey(name)) { + List types = Lists.newArrayList(); + int size = expandSizeMap.get(name); + for (int index = 0; index < size; index++) { + types.add(new TemplateType(name + "_" + index)); + } + return types; + } + return Lists.newArrayList(this); + } + @Override public String toSql(int depth) { return name; diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index c5767d6c34a3cd..655b5e5d161d4b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -524,7 +524,12 @@ public boolean hasTemplateType() { return false; } - // return a new type without template type, by specialize tempalte type in this type + // only used for struct type and variadic template type + public boolean needExpandTemplateType() { + return false; + } + + // return a new type without template type, by specialize template type in this type public Type specializeTemplateType(Type specificType, Map specializedTypeMap, boolean useSpecializedType) throws TypeException { if (hasTemplateType()) { @@ -535,6 +540,22 @@ public Type specializeTemplateType(Type specificType, Map speciali } } + /** + * Only used for struct type and variadic template type, + * collect variadic template's expand size based on the input arguments + */ + public void collectTemplateExpandSize(Type[] args, Map expandSizeMap) + throws TypeException { + } + + /** + * Only used for struct type and variadic template type, + * Do expand variadic template type + */ + public List expandVariadicTemplateType(Map expandSizeMap) { + return Lists.newArrayList(this); + } + /** * Returns true if Impala supports this type in the metdata. It does not mean we * can manipulate data of this type. For tables that contain columns with these diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index d28448730bbc5a..4526500c6e71a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -30,6 +30,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.io.output.NullOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,6 +41,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; /** @@ -847,6 +849,47 @@ public boolean hasTemplateArg() { return false; } + public boolean hasVariadicTemplateArg() { + for (Type t : getArgs()) { + if (t.needExpandTemplateType()) { + return true; + } + } + + return false; + } + + public void expandVariadicTemplate(Type[] args) throws TypeException { + // collect expand size of variadic template + Map expandSizeMap = Maps.newHashMap(); + for (int i = argTypes.length - 1; i >= 0; i--) { + if (argTypes[i].hasTemplateType()) { + if (argTypes[i].needExpandTemplateType()) { + argTypes[i].collectTemplateExpandSize( + Arrays.copyOfRange(args, i, args.length), expandSizeMap); + } + } + } + + // expand the variadic template in arg types + List newArgTypes = Lists.newArrayList(); + for (int i = 0; i < argTypes.length - 1; i++) { + if (argTypes[i].needExpandTemplateType()) { + newArgTypes.addAll(argTypes[i].expandVariadicTemplateType(expandSizeMap)); + } else { + newArgTypes.add(argTypes[i]); + } + } + argTypes = newArgTypes.toArray(new Type[0]); + + // expand the variadic template in ret type + if (retType.needExpandTemplateType()) { + List newRetType = retType.expandVariadicTemplateType(expandSizeMap); + Preconditions.checkState(newRetType.size() == 1); + retType = newRetType.get(0); + } + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index c5e8084b782735..c9b691b5d75c89 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -37,6 +37,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -1227,9 +1228,14 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is List normalFunctions = Lists.newArrayList(); List templateFunctions = Lists.newArrayList(); + List variadicTemplateFunctions = Lists.newArrayList(); for (Function fn : fns) { if (fn.hasTemplateArg()) { - templateFunctions.add(fn); + if (!fn.hasVariadicTemplateArg()) { + templateFunctions.add(fn); + } else { + variadicTemplateFunctions.add(fn); + } } else { normalFunctions.add(fn); } @@ -1244,12 +1250,29 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is // then specialize template functions and try them List specializedTemplateFunctions = Lists.newArrayList(); for (Function f : templateFunctions) { - f = FunctionSet.specializeTemplateFunction(f, desc); + f = FunctionSet.specializeTemplateFunction(f, desc, false); if (f != null) { specializedTemplateFunctions.add(f); } } - return getFunction(desc, mode, specializedTemplateFunctions); + + // try template function second + fn = getFunction(desc, mode, specializedTemplateFunctions); + if (fn != null) { + return fn; + } + + // then specialize variadic template function and try them + List specializedVariadicTemplateFunctions = Lists.newArrayList(); + for (Function f : variadicTemplateFunctions) { + f = FunctionSet.specializeTemplateFunction(f, desc, true); + if (f != null) { + specializedVariadicTemplateFunctions.add(f); + } + } + + // try variadic template function + return getFunction(desc, mode, specializedVariadicTemplateFunctions); } private Function getFunction(Function desc, Function.CompareMode mode, List fns) { @@ -1292,7 +1315,7 @@ private Function getFunction(Function desc, Function.CompareMode mode, List specializedTypeMap = Maps.newHashMap(); for (int i = 0; i < args.length; i++) { if (args[i].hasTemplateType()) { - hasTemplateType = true; args[i] = args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap, false); } } if (specializedFunction.getReturnType().hasTemplateType()) { - hasTemplateType = true; specializedFunction.setReturnType( specializedFunction.getReturnType().specializeTemplateType( requestFunction.getReturnType(), specializedTypeMap, true)); diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 222b1ba214cf36..42566fef86b089 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -74,6 +74,10 @@ [['map_keys'], 'ARRAY', ['MAP'], '', ['K', 'V']], [['map_values'], 'ARRAY', ['MAP'], '', ['K', 'V']], + # struct functions + [['struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], + [['named_struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], + # array functions [['array'], 'ARRAY', ['BOOLEAN', '...'], 'ALWAYS_NOT_NULLABLE'], [['array'], 'ARRAY', ['TINYINT', '...'], 'ALWAYS_NOT_NULLABLE'], diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index c80066bd62d06e..f7bf16dd9e859a 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -117,6 +117,8 @@ def generate_fe_datatype(str_type, template_types): # process template if str_type in template_types: return 'new TemplateType("{0}")'.format(str_type) + else if "..." + str_type in template_types: + return 'new TemplateType("{0}", true)'.format(str_type) # process Array, Map, Struct template template_start = str_type.find('<') From 2b5c5eb9339ab5819b83075b5a5149f3509a6c27 Mon Sep 17 00:00:00 2001 From: xy720 Date: Tue, 21 Mar 2023 12:31:36 +0800 Subject: [PATCH 2/5] save --- .../main/java/org/apache/doris/catalog/StructType.java | 5 +++++ .../main/java/org/apache/doris/catalog/TemplateType.java | 1 - .../src/main/java/org/apache/doris/catalog/Function.java | 8 ++++---- .../main/java/org/apache/doris/catalog/FunctionSet.java | 1 - gensrc/script/doris_builtins_functions.py | 3 +++ gensrc/script/gen_builtins_functions.py | 9 ++++++++- regression-test/conf/regression-conf.groovy | 4 ++-- 7 files changed, 22 insertions(+), 9 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java index 1d9f1c47ce1f13..fa8624a955fd44 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java @@ -31,6 +31,7 @@ import com.google.gson.annotations.SerializedName; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -70,6 +71,10 @@ public StructType(List types) { this.fields = newFields; } + public StructType(Type... types) { + this(Arrays.asList(types)); + } + public StructType() { this.fields = Lists.newArrayList(); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index 8c90834cde8fa3..4b4ca9d2d148ed 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -25,7 +25,6 @@ import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; -import java.util.Arrays; import java.util.List; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 4526500c6e71a7..771794be26bb7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -873,11 +873,11 @@ public void expandVariadicTemplate(Type[] args) throws TypeException { // expand the variadic template in arg types List newArgTypes = Lists.newArrayList(); - for (int i = 0; i < argTypes.length - 1; i++) { - if (argTypes[i].needExpandTemplateType()) { - newArgTypes.addAll(argTypes[i].expandVariadicTemplateType(expandSizeMap)); + for (Type argType : argTypes) { + if (argType.needExpandTemplateType()) { + newArgTypes.addAll(argType.expandVariadicTemplateType(expandSizeMap)); } else { - newArgTypes.add(argTypes[i]); + newArgTypes.add(argType); } } argTypes = newArgTypes.toArray(new Type[0]); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index c9b691b5d75c89..4d3c7328d916ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -37,7 +37,6 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 42566fef86b089..7f2acdd57af83d 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -78,6 +78,9 @@ [['struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], [['named_struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], + # test + [['struct'], 'STRUCT', ['T', 'TYPES'], 'ALWAYS_NOT_NULLABLE', ['T', '...TYPES']], + # array functions [['array'], 'ARRAY', ['BOOLEAN', '...'], 'ALWAYS_NOT_NULLABLE'], [['array'], 'ARRAY', ['TINYINT', '...'], 'ALWAYS_NOT_NULLABLE'], diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index f7bf16dd9e859a..15a9c1df002b81 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -54,6 +54,7 @@ \n\ import org.apache.doris.catalog.ArrayType;\n\ import org.apache.doris.catalog.MapType;\n\ +import org.apache.doris.catalog.StructType;\n\ import org.apache.doris.catalog.TemplateType;\n\ import org.apache.doris.catalog.Type;\n\ import org.apache.doris.catalog.Function;\n\ @@ -117,7 +118,7 @@ def generate_fe_datatype(str_type, template_types): # process template if str_type in template_types: return 'new TemplateType("{0}")'.format(str_type) - else if "..." + str_type in template_types: + elif "..." + str_type in template_types: return 'new TemplateType("{0}", true)'.format(str_type) # process Array, Map, Struct template @@ -131,6 +132,12 @@ def generate_fe_datatype(str_type, template_types): elif str_type.startswith("MAP<"): types = template.split(',', 2) return 'new MapType({0}, {1})'.format(generate_fe_datatype(types[0], template_types), generate_fe_datatype(types[1], template_types)) + elif str_type.startswith("STRUCT<"): + types = template.split(',') + field_str = generate_fe_datatype(types[0], template_types) + for i in range(1, len(types)): + field_str += ", " + generate_fe_datatype(types[i], template_types) + return 'new StructType({0})'.format(field_str) # lagacy Array, Map syntax if str_type.startswith("ARRAY_"): diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index ebb52dbe28faa2..8704251258810c 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -24,11 +24,11 @@ defaultDb = "regression_test" // init cmd like: select @@session.tx_read_only // at each time we connect. // add allowLoadLocalInfile so that the jdbc can execute mysql load data from client. -jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true" +jdbcUrl = "jdbc:mysql://127.0.0.1:9032/?useLocalSessionState=true&allowLoadLocalInfile=true" jdbcUser = "root" jdbcPassword = "" -feHttpAddress = "127.0.0.1:8030" +feHttpAddress = "127.0.0.1:8032" feHttpUser = "root" feHttpPassword = "" From e3a2bd8e51e74d1e8a0dec28398fdca41f6e0462 Mon Sep 17 00:00:00 2001 From: xy720 Date: Tue, 21 Mar 2023 14:03:46 +0800 Subject: [PATCH 3/5] save --- .../org/apache/doris/catalog/Function.java | 24 ++------------ .../org/apache/doris/catalog/FunctionSet.java | 33 ++++++++++++++++--- gensrc/script/doris_builtins_functions.py | 13 +++++--- gensrc/script/gen_builtins_functions.py | 3 ++ regression-test/conf/regression-conf.groovy | 4 +-- 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 771794be26bb7e..a3e3df98c812f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -30,7 +30,6 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.commons.io.output.NullOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -859,9 +858,8 @@ public boolean hasVariadicTemplateArg() { return false; } - public void expandVariadicTemplate(Type[] args) throws TypeException { - // collect expand size of variadic template - Map expandSizeMap = Maps.newHashMap(); + // collect expand size of variadic template + public void collectTemplateExpandSize(Type[] args, Map expandSizeMap) throws TypeException { for (int i = argTypes.length - 1; i >= 0; i--) { if (argTypes[i].hasTemplateType()) { if (argTypes[i].needExpandTemplateType()) { @@ -870,24 +868,6 @@ public void expandVariadicTemplate(Type[] args) throws TypeException { } } } - - // expand the variadic template in arg types - List newArgTypes = Lists.newArrayList(); - for (Type argType : argTypes) { - if (argType.needExpandTemplateType()) { - newArgTypes.addAll(argType.expandVariadicTemplateType(expandSizeMap)); - } else { - newArgTypes.add(argType); - } - } - argTypes = newArgTypes.toArray(new Type[0]); - - // expand the variadic template in ret type - if (retType.needExpandTemplateType()) { - List newRetType = retType.expandVariadicTemplateType(expandSizeMap); - Preconditions.checkState(newRetType.size() == 1); - retType = newRetType.get(0); - } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 4d3c7328d916ad..049fff4898ffed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1321,15 +1321,36 @@ public static Function specializeTemplateFunction(Function templateFunction, Fun + " return: " + templateFunction.getReturnType()); LOG.debug("requestFunction signature: " + requestFunction.signatureString() + " return: " + requestFunction.getReturnType()); - Function specializedFunction = templateFunction; + List newArgTypes = Lists.newArrayList(); + List newRetType = Lists.newArrayList(); if (isVariadic) { - templateFunction.expandVariadicTemplate(requestFunction.getArgs()); + Map expandSizeMap = Maps.newHashMap(); + templateFunction.collectTemplateExpandSize(requestFunction.getArgs(), expandSizeMap); + // expand the variadic template in arg types + for (Type argType : templateFunction.getArgs()) { + if (argType.needExpandTemplateType()) { + newArgTypes.addAll(argType.expandVariadicTemplateType(expandSizeMap)); + } else { + newArgTypes.add(argType); + } + } + + // expand the variadic template in ret type + if (templateFunction.getReturnType().needExpandTemplateType()) { + newRetType.addAll(templateFunction.getReturnType().expandVariadicTemplateType(expandSizeMap)); + Preconditions.checkState(newRetType.size() == 1); + } else { + newRetType.add(templateFunction.getReturnType()); + } + } else { + newArgTypes.addAll(Lists.newArrayList(templateFunction.getArgs())); + newRetType.add(templateFunction.getReturnType()); } + Function specializedFunction = templateFunction; if (templateFunction instanceof ScalarFunction) { ScalarFunction f = (ScalarFunction) templateFunction; - specializedFunction = new ScalarFunction(f.getFunctionName(), Lists.newArrayList(f.getArgs()), - f.getReturnType(), f.hasVarArgs(), f.getSymbolName(), f.getBinaryType(), - f.isUserVisible(), f.isVectorized(), f.getNullableMode()); + specializedFunction = new ScalarFunction(f.getFunctionName(), newArgTypes, newRetType.get(0), f.hasVarArgs(), + f.getSymbolName(), f.getBinaryType(), f.isUserVisible(), f.isVectorized(), f.getNullableMode()); } else { // TODO(xk) } @@ -1337,10 +1358,12 @@ public static Function specializeTemplateFunction(Function templateFunction, Fun Map specializedTypeMap = Maps.newHashMap(); for (int i = 0; i < args.length; i++) { if (args[i].hasTemplateType()) { + hasTemplateType = true; args[i] = args[i].specializeTemplateType(requestFunction.getArgs()[i], specializedTypeMap, false); } } if (specializedFunction.getReturnType().hasTemplateType()) { + hasTemplateType = true; specializedFunction.setReturnType( specializedFunction.getReturnType().specializeTemplateType( requestFunction.getReturnType(), specializedTypeMap, true)); diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 7f2acdd57af83d..3a5f5d7eec8864 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -37,6 +37,11 @@ # eg. [['element_at', '%element_extract%'], 'V', ['MAP', 'K'], 'ALWAYS_NULLABLE', ['K', 'V']], # 'K' and 'V' is type template and will be specialized at runtime in FE to match specific args. # +# 'template_types' support variadic template is now support variadic template. +# eg. [['struct'], 'STRUCT', ['TYPES...'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], +# Inspired by C++ std::vector::emplace_back() function. '...TYPES' is variadic template and will +# be expanded to normal templates at runtime in FE to match variadic args. Please ensure that the +# variadic template is placed at the last position of all templates. visible_functions = [ # Bit and Byte functions # For functions corresponding to builtin operators, we can reuse the implementations @@ -75,11 +80,9 @@ [['map_values'], 'ARRAY', ['MAP'], '', ['K', 'V']], # struct functions - [['struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], - [['named_struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], - - # test - [['struct'], 'STRUCT', ['T', 'TYPES'], 'ALWAYS_NOT_NULLABLE', ['T', '...TYPES']], + [['struct'], 'STRUCT', ['TYPES...'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], + [['named_struct'], 'STRUCT', ['TYPES...'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], + [['element_at', '%element_extract%'], 'VARCHAR', ['STRUCT', 'VARCHAR'], 'ALWAYS_NULLABLE', ['...TYPES']], # array functions [['array'], 'ARRAY', ['BOOLEAN', '...'], 'ALWAYS_NOT_NULLABLE'], diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index 15a9c1df002b81..851778f28dc0af 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -115,6 +115,9 @@ def generate_fe_datatype(str_type, template_types): # delete whitespace str_type = str_type.replace(' ', '').replace('\t', '') + # delete ellipsis dots + str_type = str_type.replace('...', '') + # process template if str_type in template_types: return 'new TemplateType("{0}")'.format(str_type) diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 8704251258810c..ebb52dbe28faa2 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -24,11 +24,11 @@ defaultDb = "regression_test" // init cmd like: select @@session.tx_read_only // at each time we connect. // add allowLoadLocalInfile so that the jdbc can execute mysql load data from client. -jdbcUrl = "jdbc:mysql://127.0.0.1:9032/?useLocalSessionState=true&allowLoadLocalInfile=true" +jdbcUrl = "jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true" jdbcUser = "root" jdbcPassword = "" -feHttpAddress = "127.0.0.1:8032" +feHttpAddress = "127.0.0.1:8030" feHttpUser = "root" feHttpPassword = "" From 07bc21039fed1421e36c164b4d5a040813049954 Mon Sep 17 00:00:00 2001 From: xy720 Date: Fri, 24 Mar 2023 17:20:06 +0800 Subject: [PATCH 4/5] save --- gensrc/script/doris_builtins_functions.py | 4 +--- gensrc/script/gen_builtins_functions.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 3a5f5d7eec8864..7358723a12aea4 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -80,9 +80,7 @@ [['map_values'], 'ARRAY', ['MAP'], '', ['K', 'V']], # struct functions - [['struct'], 'STRUCT', ['TYPES...'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], - [['named_struct'], 'STRUCT', ['TYPES...'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], - [['element_at', '%element_extract%'], 'VARCHAR', ['STRUCT', 'VARCHAR'], 'ALWAYS_NULLABLE', ['...TYPES']], + [['struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['TYPES...']], # array functions [['array'], 'ARRAY', ['BOOLEAN', '...'], 'ALWAYS_NOT_NULLABLE'], diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py index 851778f28dc0af..8d148e05d2da35 100755 --- a/gensrc/script/gen_builtins_functions.py +++ b/gensrc/script/gen_builtins_functions.py @@ -121,7 +121,7 @@ def generate_fe_datatype(str_type, template_types): # process template if str_type in template_types: return 'new TemplateType("{0}")'.format(str_type) - elif "..." + str_type in template_types: + elif str_type + "..." in template_types: return 'new TemplateType("{0}", true)'.format(str_type) # process Array, Map, Struct template From 084fd361367072f89b1721f94d08cd93624a6d1e Mon Sep 17 00:00:00 2001 From: xy720 Date: Fri, 24 Mar 2023 18:38:34 +0800 Subject: [PATCH 5/5] save --- gensrc/script/doris_builtins_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 7358723a12aea4..d5ad320fa48352 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -38,8 +38,8 @@ # 'K' and 'V' is type template and will be specialized at runtime in FE to match specific args. # # 'template_types' support variadic template is now support variadic template. -# eg. [['struct'], 'STRUCT', ['TYPES...'], 'ALWAYS_NOT_NULLABLE', ['...TYPES']], -# Inspired by C++ std::vector::emplace_back() function. '...TYPES' is variadic template and will +# eg. [['struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['TYPES...']], +# Inspired by C++ std::vector::emplace_back() function. 'TYPES...' is variadic template and will # be expanded to normal templates at runtime in FE to match variadic args. Please ensure that the # variadic template is placed at the last position of all templates. visible_functions = [