From bb9d3750a9ca13b1102ae834ddc4473c5c62c742 Mon Sep 17 00:00:00 2001 From: xy720 Date: Tue, 4 Apr 2023 19:47:17 +0800 Subject: [PATCH 1/6] save --- .../org/apache/doris/catalog/AnyType.java | 48 ++++++++++++++ .../apache/doris/catalog/TemplateType.java | 4 +- .../java/org/apache/doris/catalog/Type.java | 1 + .../org/apache/doris/catalog/Function.java | 4 ++ .../org/apache/doris/catalog/FunctionSet.java | 37 ++++++++++- .../doris/catalog/FunctionTypeDeducers.java | 63 +++++++++++++++++++ 6 files changed, 153 insertions(+), 4 deletions(-) create mode 100644 fe/fe-common/src/main/java/org/apache/doris/catalog/AnyType.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/AnyType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/AnyType.java new file mode 100644 index 00000000000000..7f90f463d6a460 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/AnyType.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.thrift.TColumnType; +import org.apache.doris.thrift.TTypeDesc; + +/** + * Describes a AnyType type, used for SQL function return type, + * NOT used for table column type. + */ +public class AnyType extends Type { + + @Override + protected String toSql(int depth) { + return null; + } + + @Override + protected String prettyPrint(int lpad) { + return null; + } + + @Override + public void toThrift(TTypeDesc container) { + throw new RuntimeException("can not call toThrift on AnyType."); + } + + @Override + public TColumnType toColumnTypeThrift() { + throw new RuntimeException("can not call toColumnTypeThrift on AnyType"); + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java index 4b4ca9d2d148ed..661591860105e7 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java @@ -117,8 +117,8 @@ public void collectTemplateExpandSize(Type[] args, Map expandSi expandSizeMap.computeIfAbsent(name, k -> args.length); if (expandSizeMap.get(name) != args.length) { throw new TypeException( - String.format("can not expand variadic template type %s to %s size since it's " - + "already expand as %s size", name, args.length, expandSizeMap.get(name))); + String.format("can not expand variadic template type %s to %s size since it's " + + "already expand as %s size", name, args.length, expandSizeMap.get(name))); } } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index 48037297aaf68c..3479230e176bca 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -111,6 +111,7 @@ public abstract class Type { new StructField("generic_struct", new ScalarType(PrimitiveType.NULL_TYPE)))); public static final StructType STRUCT = new StructType(); public static final VariantType VARIANT = new VariantType(); + public static final AnyType ANY_TYPE = new AnyType(); private static final Logger LOG = LogManager.getLogger(Type.class); private static final ArrayList integerTypes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 88cc1b420d33de..7ec3fa18eeab64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -493,6 +493,10 @@ private boolean isIndistinguishable(Function o) { } } + public boolean isInferenceFunction() { + return retType instanceof AnyType; + } + public TFunction toThrift(Type realReturnType, Type[] realArgTypes) { TFunction fn = new TFunction(); fn.setSignature(signatureString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 6475f3e3ec4a03..88b56ce4a39658 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1232,7 +1232,12 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is List normalFunctions = Lists.newArrayList(); List templateFunctions = Lists.newArrayList(); List variadicTemplateFunctions = Lists.newArrayList(); + List inferenceFunctions = Lists.newArrayList(); for (Function fn : fns) { + if (fn.isInferenceFunction()) { + inferenceFunctions.add(fn); + continue; + } if (fn.hasTemplateArg()) { if (!fn.hasVariadicTemplateArg()) { templateFunctions.add(fn); @@ -1274,8 +1279,25 @@ public Function getFunction(Function desc, Function.CompareMode mode, boolean is } } - // try variadic template function - return getFunction(desc, mode, specializedVariadicTemplateFunctions); + // try variadic template function third + fn = getFunction(desc, mode, specializedVariadicTemplateFunctions); + if (fn != null) { + return fn; + } + + List inferredFunctions = Lists.newArrayList(); + for (Function f : inferenceFunctions) { + if (f.hasTemplateArg()) { + f = specializeTemplateFunction(f, desc, f.hasVariadicTemplateArg()); + } + f = resolveInferenceFunction(f, desc); + if (f != null) { + inferredFunctions.add(f); + } + } + + // try inference function at last + return getFunction(desc, mode, inferredFunctions); } private Function getFunction(Function desc, Function.CompareMode mode, List fns) { @@ -1384,6 +1406,17 @@ public Function specializeTemplateFunction(Function templateFunction, Function r } } + public Function resolveInferenceFunction(Function inferenceFunction, Function requestFunction) { + Type[] args = requestFunction.getArgs(); + Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), args); + if (inferenceFunction instanceof ScalarFunction) { + ScalarFunction f = (ScalarFunction) inferenceFunction; + return new ScalarFunction(f.getFunctionName(), Lists.newArrayList(f.getArgs()), newRetType, f.hasVarArgs(), + f.getSymbolName(), f.getBinaryType(), f.isUserVisible(), f.isVectorized(), f.getNullableMode()); + } + return null; + } + /** * There are essential differences in the implementation of some functions for different * types params, which should be prohibited. diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java new file mode 100644 index 00000000000000..72b30363a44549 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; + +import java.util.List; + +public class FunctionTypeDeducers { + + public interface TypeDeducer { + public Type deduce(Type[] args); + } + + public static final ImmutableMap DEDUCERS = ImmutableMap.builder() + .put("named_struct", new NamedStructDeducer()) + .put("element_at", new ElementAtDeducer()) + .build(); + + public static Type deduce(String fnName, Type[] args) { + if (DEDUCERS.containsKey(fnName)) { + return DEDUCERS.get(fnName).deduce(args); + } + return null; + } + + public static class NamedStructDeducer implements TypeDeducer { + @Override + public Type deduce(Type[] args) { + List evenArgs = Lists.newArrayList(); + for (int i = 0; i < args.length; i++) { + if (i % 2 == 1) { + evenArgs.add(args[i]); + } + } + return new StructType(evenArgs); + } + } + + public static class ElementAtDeducer implements TypeDeducer { + @Override + public Type deduce(Type[] args) { + // todo(xy) + return null; + } + } +} From 37976db3ced63715949347e83817862b2549ee72 Mon Sep 17 00:00:00 2001 From: xy720 Date: Tue, 4 Apr 2023 19:53:54 +0800 Subject: [PATCH 2/6] save --- gensrc/script/doris_builtins_functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 3d175d0fa32cfa..4024cdab6ae5b6 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -81,6 +81,7 @@ # struct functions [['struct'], 'STRUCT', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['TYPES...']], + [['named_struct'], 'ANY_TYPE', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['TYPES...']], # array functions [['array'], 'ARRAY', ['BOOLEAN', '...'], 'ALWAYS_NOT_NULLABLE'], From ccee1d8a99d3a01a009d1ca377a6e170280b9681 Mon Sep 17 00:00:00 2001 From: xy720 Date: Tue, 4 Apr 2023 20:10:19 +0800 Subject: [PATCH 3/6] save --- .../src/main/java/org/apache/doris/catalog/FunctionSet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 88b56ce4a39658..be8d9444231bff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1409,7 +1409,7 @@ public Function specializeTemplateFunction(Function templateFunction, Function r public Function resolveInferenceFunction(Function inferenceFunction, Function requestFunction) { Type[] args = requestFunction.getArgs(); Type newRetType = FunctionTypeDeducers.deduce(inferenceFunction.functionName(), args); - if (inferenceFunction instanceof ScalarFunction) { + if (newRetType != null && inferenceFunction instanceof ScalarFunction) { ScalarFunction f = (ScalarFunction) inferenceFunction; return new ScalarFunction(f.getFunctionName(), Lists.newArrayList(f.getArgs()), newRetType, f.hasVarArgs(), f.getSymbolName(), f.getBinaryType(), f.isUserVisible(), f.isVectorized(), f.getNullableMode()); From 1c97ea336868e77ff3b7072d094741d05dca9b77 Mon Sep 17 00:00:00 2001 From: xy720 Date: Wed, 5 Apr 2023 02:26:47 +0800 Subject: [PATCH 4/6] save --- .../org/apache/doris/catalog/StructType.java | 9 +++++++ .../doris/analysis/FunctionCallExpr.java | 24 +++++++++++++++++++ .../doris/catalog/FunctionTypeDeducers.java | 2 +- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java index a037b77de9afc2..bbf6a87f01cc4e 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java @@ -276,6 +276,15 @@ public List expandVariadicTemplateType(Map expandSizeMap) return Lists.newArrayList(this); } + public StructType replaceFieldsWithNames(List names) { + Preconditions.checkState(names.size() == fields.size()); + ArrayList newFields = Lists.newArrayList(); + for (int i = 0; i < names.size(); i++) { + newFields.add(new StructField(names.get(i), fields.get(i).type)); + } + return new StructType(newFields); + } + @Override public boolean equals(Object other) { if (!(other instanceof StructType)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 0dbbe6f62a0c17..cbaacc8038163e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1448,6 +1448,21 @@ && collectChildReturnTypes()[0].isDecimalV3()) { fn.getReturnType().getPrimitiveType().setTimeType(); } + if (fnName.getFunction().equalsIgnoreCase("named_struct")) { + if ((children.size() & 1) == 1) { + throw new AnalysisException("named_struct can't be odd parameters, need even parameters: " + + this.toSql()); + } + for (int i = 0; i < children.size(); i++) { + if ((i & 1) == 0) { + if (!(getChild(i) instanceof StringLiteral)) { + throw new AnalysisException( + "named_struct only allows constant string parameter in odd position: " + this.toSql()); + } + } + } + } + if (isAggregateFunction()) { final String functionName = fnName.getFunction(); // subexprs must not contain aggregates @@ -1612,6 +1627,15 @@ private void analyzeNestedFunction() { if (children.size() > 1) { this.type = new MapType(children.get(0).getType(), children.get(1).getType()); } + } else if (fnName.getFunction().equalsIgnoreCase("named_struct")) { + List fieldNames = Lists.newArrayList(); + for (int i = 0; i < children.size(); i++) { + if ((i & 1) == 0) { + StringLiteral nameLiteral = (StringLiteral) children.get(i); + fieldNames.add(nameLiteral.getStringValue()); + } + } + this.type = ((StructType) type).replaceFieldsWithNames(fieldNames); } else if (fnName.getFunction().equalsIgnoreCase("if")) { if (children.get(1).getType().isArrayType() && ( ((ArrayType) children.get(1).getType()).getItemType().isDecimalV3() diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java index 72b30363a44549..0e547cdcc32595 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java @@ -45,7 +45,7 @@ public static class NamedStructDeducer implements TypeDeducer { public Type deduce(Type[] args) { List evenArgs = Lists.newArrayList(); for (int i = 0; i < args.length; i++) { - if (i % 2 == 1) { + if ((i & 1) == 1) { evenArgs.add(args[i]); } } From 7beed8ea0ef30f3cdc86b89605baaeb43e5e0fc9 Mon Sep 17 00:00:00 2001 From: xy720 Date: Thu, 6 Apr 2023 16:12:21 +0800 Subject: [PATCH 5/6] save --- .../src/main/java/org/apache/doris/catalog/Function.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java index 7ec3fa18eeab64..ce3f77cb094e47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java @@ -494,6 +494,11 @@ private boolean isIndistinguishable(Function o) { } public boolean isInferenceFunction() { + for (Type arg : argTypes) { + if (arg instanceof AnyType) { + return true; + } + } return retType instanceof AnyType; } From 6f6f19ce6a529f2d3f0628b5db03d5d0c2be2848 Mon Sep 17 00:00:00 2001 From: xy720 Date: Thu, 6 Apr 2023 16:15:14 +0800 Subject: [PATCH 6/6] save --- .../doris/analysis/FunctionCallExpr.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index cbaacc8038163e..0dbbe6f62a0c17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1448,21 +1448,6 @@ && collectChildReturnTypes()[0].isDecimalV3()) { fn.getReturnType().getPrimitiveType().setTimeType(); } - if (fnName.getFunction().equalsIgnoreCase("named_struct")) { - if ((children.size() & 1) == 1) { - throw new AnalysisException("named_struct can't be odd parameters, need even parameters: " - + this.toSql()); - } - for (int i = 0; i < children.size(); i++) { - if ((i & 1) == 0) { - if (!(getChild(i) instanceof StringLiteral)) { - throw new AnalysisException( - "named_struct only allows constant string parameter in odd position: " + this.toSql()); - } - } - } - } - if (isAggregateFunction()) { final String functionName = fnName.getFunction(); // subexprs must not contain aggregates @@ -1627,15 +1612,6 @@ private void analyzeNestedFunction() { if (children.size() > 1) { this.type = new MapType(children.get(0).getType(), children.get(1).getType()); } - } else if (fnName.getFunction().equalsIgnoreCase("named_struct")) { - List fieldNames = Lists.newArrayList(); - for (int i = 0; i < children.size(); i++) { - if ((i & 1) == 0) { - StringLiteral nameLiteral = (StringLiteral) children.get(i); - fieldNames.add(nameLiteral.getStringValue()); - } - } - this.type = ((StructType) type).replaceFieldsWithNames(fieldNames); } else if (fnName.getFunction().equalsIgnoreCase("if")) { if (children.get(1).getType().isArrayType() && ( ((ArrayType) children.get(1).getType()).getItemType().isDecimalV3()