From 775b55e360fbfc58c121e64ba8e1322e94758ac0 Mon Sep 17 00:00:00 2001 From: Mryange <2319153948@qq.com> Date: Thu, 20 Jul 2023 16:32:03 +0800 Subject: [PATCH 1/4] update1 --- be/src/vec/functions/function_java_udf.cpp | 53 +++- be/src/vec/functions/function_java_udf.h | 1 + .../doris/common/jni/utils/UdfUtils.java | 48 +++- .../org/apache/doris/udf/BaseExecutor.java | 234 ++++++++++++++++++ .../org/apache/doris/udf/UdfExecutor.java | 172 ++++++++++++- .../org/apache/doris/catalog/MapType.java | 9 + .../java/org/apache/doris/catalog/Type.java | 2 + .../doris/analysis/CreateFunctionStmt.java | 4 + .../org/apache/doris/catalog/ColumnType.java | 15 +- 9 files changed, 529 insertions(+), 9 deletions(-) diff --git a/be/src/vec/functions/function_java_udf.cpp b/be/src/vec/functions/function_java_udf.cpp index cd992b84e10e2b..0df2806026d68e 100644 --- a/be/src/vec/functions/function_java_udf.cpp +++ b/be/src/vec/functions/function_java_udf.cpp @@ -32,6 +32,7 @@ #include "util/jni-util.h" #include "vec/columns/column.h" #include "vec/columns/column_array.h" +#include "vec/columns/column_map.h" #include "vec/columns/column_nullable.h" #include "vec/columns/column_string.h" #include "vec/columns/column_vector.h" @@ -71,7 +72,8 @@ Status JavaFunctionCall::open(FunctionContext* context, FunctionContext::Functio jni_env->executor_cl, "convertBasicArguments", "(IZIJJJ)[Ljava/lang/Object;"); jni_env->executor_convert_array_argument_id = env->GetMethodID( jni_env->executor_cl, "convertArrayArguments", "(IZIJJJJJ)[Ljava/lang/Object;"); - + jni_env->executor_convert_map_argument_id = env->GetMethodID( + jni_env->executor_cl, "convertMapArguments", "(IZIJJJJJJJJ)[Ljava/lang/Object;"); jni_env->executor_result_basic_batch_id = env->GetMethodID( jni_env->executor_cl, "copyBatchBasicResult", "(ZI[Ljava/lang/Object;JJJ)V"); jni_env->executor_result_array_batch_id = env->GetMethodID( @@ -148,6 +150,7 @@ Status JavaFunctionCall::execute(FunctionContext* context, Block& block, ColumnPtr null_cols[arg_size]; jclass obj_class = env->FindClass("[Ljava/lang/Object;"); jclass arraylist_class = env->FindClass("Ljava/util/ArrayList;"); + // jclass hashmap_class = env->FindClass("Ljava/util/HashMap;"); jobjectArray arg_objects = env->NewObjectArray(arg_size, obj_class, nullptr); int64_t nullmap_address = 0; for (size_t arg_idx = 0; arg_idx < arg_size; ++arg_idx) { @@ -218,6 +221,54 @@ Status JavaFunctionCall::execute(FunctionContext* context, Block& block, jni_env->executor_convert_array_argument_id, arg_idx, arg_column_nullable, num_rows, nullmap_address, offset_address, nested_nullmap_address, nested_data_address, nested_offset_address); + } else if (data_cols[arg_idx]->is_column_map()) { + const ColumnMap* map_col = assert_cast(data_cols[arg_idx].get()); + auto offset_address = + reinterpret_cast(map_col->get_offsets_column().get_raw_data().data); + const ColumnNullable& map_key_column_nullable = + assert_cast(map_col->get_keys()); + auto key_data_column_null_map = map_key_column_nullable.get_null_map_column_ptr(); + auto key_data_column = map_key_column_nullable.get_nested_column_ptr(); + + auto key_nested_nullmap_address = reinterpret_cast( + check_and_get_column>(key_data_column_null_map) + ->get_data() + .data()); + int64_t key_nested_data_address = 0, key_nested_offset_address = 0; + if (key_data_column->is_column_string()) { + const ColumnString* col = assert_cast(key_data_column.get()); + key_nested_data_address = reinterpret_cast(col->get_chars().data()); + key_nested_offset_address = reinterpret_cast(col->get_offsets().data()); + } else { + key_nested_data_address = + reinterpret_cast(key_data_column->get_raw_data().data); + } + + const ColumnNullable& map_value_column_nullable = + assert_cast(map_col->get_values()); + auto value_data_column_null_map = map_value_column_nullable.get_null_map_column_ptr(); + auto value_data_column = map_value_column_nullable.get_nested_column_ptr(); + auto value_nested_nullmap_address = reinterpret_cast( + check_and_get_column>(value_data_column_null_map) + ->get_data() + .data()); + int64_t value_nested_data_address = 0, value_nested_offset_address = 0; + // array type need pass address: [nullmap_address], offset_address, nested_nullmap_address, nested_data_address/nested_char_address,nested_offset_address + if (value_data_column->is_column_string()) { + const ColumnString* col = assert_cast(value_data_column.get()); + value_nested_data_address = reinterpret_cast(col->get_chars().data()); + value_nested_offset_address = reinterpret_cast(col->get_offsets().data()); + } else { + value_nested_data_address = + reinterpret_cast(value_data_column->get_raw_data().data); + } + arr_obj = (jobjectArray)env->CallNonvirtualObjectMethod( + jni_ctx->executor, jni_env->executor_cl, + jni_env->executor_convert_map_argument_id, arg_idx, arg_column_nullable, + num_rows, nullmap_address, offset_address, key_nested_nullmap_address, + key_nested_data_address, key_nested_offset_address, + value_nested_nullmap_address, value_nested_data_address, + value_nested_offset_address); } else { return Status::InvalidArgument( strings::Substitute("Java UDF doesn't support type $0 now !", diff --git a/be/src/vec/functions/function_java_udf.h b/be/src/vec/functions/function_java_udf.h index ba17942ccedff8..4398fa038d23d2 100644 --- a/be/src/vec/functions/function_java_udf.h +++ b/be/src/vec/functions/function_java_udf.h @@ -99,6 +99,7 @@ class JavaFunctionCall : public IFunctionBase { jmethodID executor_evaluate_id; jmethodID executor_convert_basic_argument_id; jmethodID executor_convert_array_argument_id; + jmethodID executor_convert_map_argument_id; jmethodID executor_result_basic_batch_id; jmethodID executor_result_array_batch_id; jmethodID executor_close_id; diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java index 09e0dc380462f8..c546750bcfbcbb 100644 --- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java @@ -18,6 +18,7 @@ package org.apache.doris.common.jni.utils; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; @@ -52,7 +53,7 @@ import java.util.Set; public class UdfUtils { - private static final Logger LOG = Logger.getLogger(UdfUtils.class); + public static final Logger LOG = Logger.getLogger(UdfUtils.class); public static final Unsafe UNSAFE; private static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L; public static final long BYTE_ARRAY_OFFSET; @@ -95,15 +96,16 @@ public enum JavaUdfDataType { DECIMAL32("DECIMAL32", TPrimitiveType.DECIMAL32, 4), DECIMAL64("DECIMAL64", TPrimitiveType.DECIMAL64, 8), DECIMAL128("DECIMAL128", TPrimitiveType.DECIMAL128I, 16), - ARRAY_TYPE("ARRAY_TYPE", TPrimitiveType.ARRAY, 0); - + ARRAY_TYPE("ARRAY_TYPE", TPrimitiveType.ARRAY, 0), + MAP_TYPE("MAP_TYPE", TPrimitiveType.MAP, 0); private final String description; private final TPrimitiveType thriftType; private final int len; private int precision; private int scale; private Type itemType; - + private Type keyType; + private Type valueType; JavaUdfDataType(String description, TPrimitiveType thriftType, int len) { this.description = description; this.thriftType = thriftType; @@ -153,6 +155,8 @@ public static Set getCandidateTypes(Class c) { JavaUdfDataType.DECIMAL128); } else if (c == java.util.ArrayList.class) { return Sets.newHashSet(JavaUdfDataType.ARRAY_TYPE); + } else if (c == java.util.HashMap.class) { + return Sets.newHashSet(JavaUdfDataType.MAP_TYPE); } return Sets.newHashSet(JavaUdfDataType.INVALID_TYPE); } @@ -192,6 +196,22 @@ public Type getItemType() { public void setItemType(Type type) { this.itemType = type; } + + public Type getKeyType() { + return keyType; + } + + public Type getValueType() { + return valueType; + } + + public void setKeyType(Type type) { + this.keyType = type; + } + + public void setValueType(Type type) { + this.valueType = type; + } } public static Pair fromThrift(TTypeDesc typeDesc, int nodeIdx) throws InternalException { @@ -232,6 +252,14 @@ public static Pair fromThrift(TTypeDesc typeDesc, int nodeIdx) th nodeIdx = childType.second; break; } + case MAP: { + Preconditions.checkState(nodeIdx + 1 < typeDesc.getTypesSize()); + Pair keyType = fromThrift(typeDesc, nodeIdx + 1); + Pair valueType = fromThrift(typeDesc, nodeIdx + 1 + keyType.value()); + type = new MapType(keyType.key(), valueType.key()); + nodeIdx = 1 + keyType.value() + valueType.value(); + break; + } default: throw new InternalException("Return type " + node.getType() + " is not supported now!"); @@ -307,6 +335,14 @@ public static Pair setReturnType(Type retType, Class setArgTypes(Type[] parameterTypes } else if (parameterTypes[finalI].isArrayType()) { ArrayType arrType = (ArrayType) parameterTypes[finalI]; inputArgTypes[i].setItemType(arrType.getItemType()); + } else if (parameterTypes[finalI].isMapType()) { + MapType mapType = (MapType) parameterTypes[finalI]; + inputArgTypes[i].setKeyType(mapType.getKeyType()); + inputArgTypes[i].setValueType(mapType.getValueType()); } if (res.length == 0) { return Pair.of(false, inputArgTypes); diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java index ef405197d63f48..8cba4dc5100a60 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java @@ -1202,4 +1202,238 @@ public Object[] convertArrayArg(int argIdx, boolean isNullable, int rowStart, in } return argument; } + + public Object[] convertMapKeyArg(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, + long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { + Object[] argument = (Object[]) Array.newInstance(ArrayList.class, rowEnd - rowStart); + for (int row = rowStart; row < rowEnd; ++row) { + long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1)); + long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row)); + int currentRowNum = (int) (offsetEnd - offsetStart); + switch (argTypes[argIdx].getKeyType().getPrimitiveType()) { + case BOOLEAN: { + argument[row - rowStart] = UdfConvert + .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case TINYINT: { + argument[row - rowStart] = UdfConvert + .convertArrayTinyIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case SMALLINT: { + argument[row - rowStart] = UdfConvert + .convertArraySmallIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case INT: { + argument[row - rowStart] = UdfConvert + .convertArrayIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case BIGINT: { + argument[row - rowStart] = UdfConvert + .convertArrayBigIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case LARGEINT: { + argument[row - rowStart] = UdfConvert + .convertArrayLargeIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case FLOAT: { + argument[row - rowStart] = UdfConvert + .convertArrayFloatArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DOUBLE: { + argument[row - rowStart] = UdfConvert + .convertArrayDoubleArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case CHAR: + case VARCHAR: + case STRING: { + argument[row - rowStart] = UdfConvert + .convertArrayStringArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr, strOffsetAddr); + break; + } + case DATE: { + argument[row - rowStart] = UdfConvert + .convertArrayDateArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DATETIME: { + argument[row - rowStart] = UdfConvert + .convertArrayDateTimeArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DATEV2: { + argument[row - rowStart] = UdfConvert + .convertArrayDateV2Arg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DATETIMEV2: { + argument[row - rowStart] = UdfConvert + .convertArrayDateTimeV2Arg(row, currentRowNum, offsetStart, isNullable, + nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + case DECIMALV2: + case DECIMAL128: { + argument[row - rowStart] = UdfConvert + .convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, row, currentRowNum, + offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + case DECIMAL32: { + argument[row - rowStart] = UdfConvert + .convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, row, currentRowNum, + offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + case DECIMAL64: { + argument[row - rowStart] = UdfConvert + .convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, row, currentRowNum, + offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + default: { + LOG.info("Not support: " + argTypes[argIdx]); + Preconditions.checkState(false, "Not support type " + argTypes[argIdx].toString()); + break; + } + } + } + return argument; + } + + public Object[] convertMapValueArg(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, + long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { + Object[] argument = (Object[]) Array.newInstance(ArrayList.class, rowEnd - rowStart); + for (int row = rowStart; row < rowEnd; ++row) { + long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1)); + long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row)); + int currentRowNum = (int) (offsetEnd - offsetStart); + switch (argTypes[argIdx].getValueType().getPrimitiveType()) { + case BOOLEAN: { + argument[row - rowStart] = UdfConvert + .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case TINYINT: { + argument[row - rowStart] = UdfConvert + .convertArrayTinyIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case SMALLINT: { + argument[row - rowStart] = UdfConvert + .convertArraySmallIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case INT: { + argument[row - rowStart] = UdfConvert + .convertArrayIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case BIGINT: { + argument[row - rowStart] = UdfConvert + .convertArrayBigIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case LARGEINT: { + argument[row - rowStart] = UdfConvert + .convertArrayLargeIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case FLOAT: { + argument[row - rowStart] = UdfConvert + .convertArrayFloatArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DOUBLE: { + argument[row - rowStart] = UdfConvert + .convertArrayDoubleArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case CHAR: + case VARCHAR: + case STRING: { + argument[row - rowStart] = UdfConvert + .convertArrayStringArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr, strOffsetAddr); + break; + } + case DATE: { + argument[row - rowStart] = UdfConvert + .convertArrayDateArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DATETIME: { + argument[row - rowStart] = UdfConvert + .convertArrayDateTimeArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DATEV2: { + argument[row - rowStart] = UdfConvert + .convertArrayDateV2Arg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); + break; + } + case DATETIMEV2: { + argument[row - rowStart] = UdfConvert + .convertArrayDateTimeV2Arg(row, currentRowNum, offsetStart, isNullable, + nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + case DECIMALV2: + case DECIMAL128: { + argument[row - rowStart] = UdfConvert + .convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, row, currentRowNum, + offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + case DECIMAL32: { + argument[row - rowStart] = UdfConvert + .convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, row, currentRowNum, + offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + case DECIMAL64: { + argument[row - rowStart] = UdfConvert + .convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, row, currentRowNum, + offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); + break; + } + default: { + LOG.info("Not support: " + argTypes[argIdx]); + Preconditions.checkState(false, "Not support type " + argTypes[argIdx].toString()); + break; + } + } + } + return argument; + } } diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java index 50528d007b0fab..09c8c3a3960671 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java @@ -17,6 +17,7 @@ package org.apache.doris.udf; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; import org.apache.doris.common.exception.UdfRuntimeException; @@ -36,14 +37,20 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.net.MalformedURLException; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashMap; public class UdfExecutor extends BaseExecutor { - private static final Logger LOG = Logger.getLogger(UdfExecutor.class); + // private static final java.util.logging.Logger LOG = + // Logger.getLogger(UdfExecutor.class); + public static final Logger LOG = Logger.getLogger(UdfExecutor.class); // setup by init() and cleared by close() private Method method; - // Pre-constructed input objects for the UDF. This minimizes object creation overhead + // Pre-constructed input objects for the UDF. This minimizes object creation + // overhead // as these objects are reused across calls to evaluate(). private Object[] inputObjects; @@ -119,13 +126,96 @@ public Object[] convertBasicArguments(int argIdx, boolean isNullable, int numRow return convertBasicArg(true, argIdx, isNullable, 0, numRows, nullMapAddr, columnAddr, strOffsetAddr); } - public Object[] convertArrayArguments(int argIdx, boolean isNullable, int numRows, long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { return convertArrayArg(argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, nestedNullMapAddr, dataAddr, strOffsetAddr); } + public Object[] convertMapArguments(int argIdx, boolean isNullable, int numRows, long nullMapAddr, + long offsetsAddr, long keyNestedNullMapAddr, long keyDataAddr, long keyStrOffsetAddr, + long valueNestedNullMapAddr, long valueDataAddr, long valueStrOffsetAddr) { + Object[] keyCol = convertMapKeyArg(argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, + keyNestedNullMapAddr, keyDataAddr, + keyStrOffsetAddr); + Object[] valueCol = convertMapValueArg(argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, + valueNestedNullMapAddr, valueDataAddr, + valueStrOffsetAddr); + PrimitiveType keyType = argTypes[argIdx].getKeyType().getPrimitiveType(); + PrimitiveType valueType = argTypes[argIdx].getValueType().getPrimitiveType(); + if (keyType == PrimitiveType.INT && valueType == PrimitiveType.INT) { + Object[] retHashMap = new HashMap[keyCol.length]; + for (int colIdx = 0; colIdx < keyCol.length; colIdx++) { + HashMap hashMap = new HashMap<>(); + ArrayList keys = (ArrayList) (keyCol[colIdx]); + ArrayList values = (ArrayList) (valueCol[colIdx]); + for (int i = 0; i < keys.size(); i++) { + Integer key = keys.get(i); + Integer value = values.get(i); + hashMap.put(key, value); + LOG.info("yxc test hashmap " + "key : " + key + " value : " + value); + } + LOG.info("yxc test colIdx : " + colIdx + " size:" + hashMap.size()); + retHashMap[colIdx] = hashMap; + } + return retHashMap; + } + if (keyType == PrimitiveType.STRING && valueType == PrimitiveType.STRING) { + return new BuildMapFromType().get(keyCol, valueCol); + } + switch (keyType) { + case BOOLEAN: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case TINYINT: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case SMALLINT: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case INT: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case BIGINT: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case LARGEINT: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case FLOAT: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case DOUBLE: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case CHAR: + case VARCHAR: + case STRING: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case DATEV2: + case DATE: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case DATETIMEV2: + case DATETIME: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + case DECIMAL32: + case DECIMAL64: + case DECIMALV2: + case DECIMAL128: { + return new HashMapBuilder().get(keyCol, valueCol, valueType); + } + default: { + LOG.info("Not support: " + keyType); + Preconditions.checkState(false, "Not support type " + keyType.toString()); + break; + } + } + return null; + } + /** * Evaluates the UDF with 'args' as the input to the UDF. */ @@ -503,4 +593,80 @@ protected void init(TJavaUdfExecutorCtorParams request, String jarPath, Type fun throw new UdfRuntimeException("Unable to call create UDF instance.", e); } } + + public static class HashMapBuilder { + public Object[] get(Object[] keyCol, Object[] valueCol, PrimitiveType valueType) { + switch (valueType) { + case BOOLEAN: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case TINYINT: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case SMALLINT: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case INT: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case BIGINT: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case LARGEINT: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case FLOAT: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case DOUBLE: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case CHAR: + case VARCHAR: + case STRING: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case DATEV2: + case DATE: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case DATETIMEV2: + case DATETIME: { + return new BuildMapFromType().get(keyCol, valueCol); + } + case DECIMAL32: + case DECIMAL64: + case DECIMALV2: + case DECIMAL128: { + return new BuildMapFromType().get(keyCol, valueCol); + } + default: { + LOG.info("Not support: " + valueType); + Preconditions.checkState(false, "Not support type " + valueType.toString()); + break; + } + } + return null; + } + } + + public static class BuildMapFromType { + public Object[] get(Object[] keyCol, Object[] valueCol) { + Object[] retHashMap = new HashMap[keyCol.length]; + for (int colIdx = 0; colIdx < keyCol.length; colIdx++) { + HashMap hashMap = new HashMap<>(); + ArrayList keys = (ArrayList) (keyCol[colIdx]); + ArrayList values = (ArrayList) (valueCol[colIdx]); + for (int i = 0; i < keys.size(); i++) { + T1 key = keys.get(i); + T2 value = values.get(i); + hashMap.put(key, value); + LOG.info("yxc test hashmap " + "key : " + key + " value : " + value); + } + LOG.info("yxc test colIdx : " + colIdx + " size:" + hashMap.size()); + retHashMap[colIdx] = hashMap; + } + return retHashMap; + } + } } diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java index 8b7a83bc936a91..82277706360b0f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java @@ -63,6 +63,15 @@ public MapType(Type keyType, Type valueType) { this.isValueContainsNull = true; } + public MapType(Type keyType, Type valueType, boolean keyContainsNull, boolean valueContainsNull) { + Preconditions.checkNotNull(keyType); + Preconditions.checkNotNull(valueType); + this.keyType = keyType; + this.isKeyContainsNull = keyContainsNull; + this.valueType = valueType; + this.isValueContainsNull = valueContainsNull; + } + @Override public PrimitiveType getPrimitiveType() { return PrimitiveType.MAP; diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java index db3bb398ab2415..c57ad049586f93 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java @@ -38,6 +38,7 @@ import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -258,6 +259,7 @@ public abstract class Type { .put(PrimitiveType.DECIMAL64, Sets.newHashSet(BigDecimal.class)) .put(PrimitiveType.DECIMAL128, Sets.newHashSet(BigDecimal.class)) .put(PrimitiveType.ARRAY, Sets.newHashSet(ArrayList.class)) + .put(PrimitiveType.MAP, Sets.newHashSet(HashMap.class)) .build(); public static ArrayList getIntegerTypes() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java index 06422e9b357e0c..d00318d0a27529 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.MapType; import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; @@ -577,6 +578,9 @@ private void checkUdfType(Class clazz, Method method, Type expType, Class pType, } else if (expType instanceof ArrayType) { ArrayType arrayType = (ArrayType) expType; javaTypes = Type.PrimitiveTypeToJavaClassType.get(arrayType.getPrimitiveType()); + } else if (expType instanceof MapType) { + MapType mapType = (MapType) expType; + javaTypes = Type.PrimitiveTypeToJavaClassType.get(mapType.getPrimitiveType()); } else { throw new AnalysisException( String.format("Method '%s' in class '%s' does not support type '%s'", diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java index 10f88eb27f39c8..d4813dbc8200ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -166,7 +166,7 @@ static boolean isSchemaChangeAllowed(Type lhs, Type rhs) { } public static void write(DataOutput out, Type type) throws IOException { - Preconditions.checkArgument(type.isScalarType() || type.isArrayType(), + Preconditions.checkArgument(type.isScalarType() || type.isArrayType() || type.isMapType(), "only support scalar type and array serialization"); if (type.isScalarType()) { ScalarType scalarType = (ScalarType) type; @@ -181,6 +181,13 @@ public static void write(DataOutput out, Type type) throws IOException { Text.writeString(out, arrayType.getPrimitiveType().name()); write(out, arrayType.getItemType()); out.writeBoolean(arrayType.getContainsNull()); + } else if (type.isMapType()) { + MapType mapType = (MapType) type; + Text.writeString(out, mapType.getPrimitiveType().name()); + write(out, mapType.getKeyType()); + write(out, mapType.getValueType()); + out.writeBoolean(mapType.getIsKeyContainsNull()); + out.writeBoolean(mapType.getIsValueContainsNull()); } } @@ -190,6 +197,12 @@ public static Type read(DataInput in) throws IOException { Type itermType = read(in); boolean containsNull = in.readBoolean(); return ArrayType.create(itermType, containsNull); + } else if (primitiveType == PrimitiveType.MAP) { + Type keyType = read(in); + Type valueType = read(in); + boolean keyContainsNull = in.readBoolean(); + boolean valueContainsNull = in.readBoolean(); + return new MapType(keyType, valueType, keyContainsNull, valueContainsNull); } else { int scale = in.readInt(); int precision = in.readInt(); From 19040edb3977a959a3874b6636af00875f31f93c Mon Sep 17 00:00:00 2001 From: Mryange <2319153948@qq.com> Date: Fri, 21 Jul 2023 11:24:05 +0800 Subject: [PATCH 2/4] add case --- .../org/apache/doris/udf/UdfExecutor.java | 22 ----- .../data/javaudf_p0/test_javaudf_map.out | 10 +++ .../org/apache/doris/udf/MapIntIntTest.java | 35 ++++++++ .../org/apache/doris/udf/MapStrStrTest.java | 36 ++++++++ .../suites/javaudf_p0/test_javaudf_map.groovy | 86 +++++++++++++++++++ 5 files changed, 167 insertions(+), 22 deletions(-) create mode 100644 regression-test/data/javaudf_p0/test_javaudf_map.out create mode 100644 regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapIntIntTest.java create mode 100644 regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapStrStrTest.java create mode 100644 regression-test/suites/javaudf_p0/test_javaudf_map.groovy diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java index 09c8c3a3960671..7af24809bbc2bf 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java @@ -143,26 +143,6 @@ public Object[] convertMapArguments(int argIdx, boolean isNullable, int numRows, valueStrOffsetAddr); PrimitiveType keyType = argTypes[argIdx].getKeyType().getPrimitiveType(); PrimitiveType valueType = argTypes[argIdx].getValueType().getPrimitiveType(); - if (keyType == PrimitiveType.INT && valueType == PrimitiveType.INT) { - Object[] retHashMap = new HashMap[keyCol.length]; - for (int colIdx = 0; colIdx < keyCol.length; colIdx++) { - HashMap hashMap = new HashMap<>(); - ArrayList keys = (ArrayList) (keyCol[colIdx]); - ArrayList values = (ArrayList) (valueCol[colIdx]); - for (int i = 0; i < keys.size(); i++) { - Integer key = keys.get(i); - Integer value = values.get(i); - hashMap.put(key, value); - LOG.info("yxc test hashmap " + "key : " + key + " value : " + value); - } - LOG.info("yxc test colIdx : " + colIdx + " size:" + hashMap.size()); - retHashMap[colIdx] = hashMap; - } - return retHashMap; - } - if (keyType == PrimitiveType.STRING && valueType == PrimitiveType.STRING) { - return new BuildMapFromType().get(keyCol, valueCol); - } switch (keyType) { case BOOLEAN: { return new HashMapBuilder().get(keyCol, valueCol, valueType); @@ -661,9 +641,7 @@ public Object[] get(Object[] keyCol, Object[] valueCol) { T1 key = keys.get(i); T2 value = values.get(i); hashMap.put(key, value); - LOG.info("yxc test hashmap " + "key : " + key + " value : " + value); } - LOG.info("yxc test colIdx : " + colIdx + " size:" + hashMap.size()); retHashMap[colIdx] = hashMap; } return retHashMap; diff --git a/regression-test/data/javaudf_p0/test_javaudf_map.out b/regression-test/data/javaudf_p0/test_javaudf_map.out new file mode 100644 index 00000000000000..7c7cf58b2f038c --- /dev/null +++ b/regression-test/data/javaudf_p0/test_javaudf_map.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_1 -- +{1:1, 10:1, 100:1} 111 +{2:1, 20:1, 200:1, 2000:1} 2222 +{3:1} 3 + +-- !select_2 -- +{"114":"514", "1919":"810"} 1145141919810 +{"a":"bc", "def":"g", "hij":"k"} abcdefghijk + diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapIntIntTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapIntIntTest.java new file mode 100644 index 00000000000000..ae1fb9bc77cd84 --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapIntIntTest.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import org.apache.hadoop.hive.ql.exec.UDF; + +import java.util.*; + +public class MapIntIntTest extends UDF { + public Integer evaluate(HashMap hashMap) { + Integer mul = 0; + for (Map.Entry entry : hashMap.entrySet()) { + Integer key = entry.getKey(); + Integer value = entry.getValue(); + mul += key * value; + } + return mul; + } +} + diff --git a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapStrStrTest.java b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapStrStrTest.java new file mode 100644 index 00000000000000..fd5ac9c20f203e --- /dev/null +++ b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/MapStrStrTest.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.udf; + +import org.apache.hadoop.hive.ql.exec.UDF; + +import java.util.*; + +public class MapStrStrTest extends UDF { + public String evaluate(HashMap hashMap) { + StringBuffer sb = new StringBuffer(); + for (Map.Entry entry : hashMap.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + sb.append((key + value)); + } + String ans = sb.toString(); + return ans; + } +} + diff --git a/regression-test/suites/javaudf_p0/test_javaudf_map.groovy b/regression-test/suites/javaudf_p0/test_javaudf_map.groovy new file mode 100644 index 00000000000000..8ca8f111f5a91b --- /dev/null +++ b/regression-test/suites/javaudf_p0/test_javaudf_map.groovy @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_javaudf_map") { + def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" + sql """ admin set frontend config("enable_map_type" = "true"); """ + log.info("Jar path: ${jarPath}".toString()) + try { + try_sql("DROP FUNCTION IF EXISTS udfii(Map);") + try_sql("DROP FUNCTION IF EXISTS udfss(Map);") + try_sql("DROP TABLE IF EXISTS map_ii") + try_sql("DROP TABLE IF EXISTS map_ss") + sql """ + CREATE TABLE IF NOT EXISTS map_ii ( + `id` INT(11) NULL COMMENT "", + `m` Map NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ); + """ + sql """ """ + sql """ INSERT INTO map_ii VALUES(1, {1:1,10:1,100:1}); """ + sql """ INSERT INTO map_ii VALUES(2, {2:1,20:1,200:1,2000:1}); """ + sql """ INSERT INTO map_ii VALUES(3, {3:1}); """ + sql """ DROP FUNCTION IF EXISTS udfii(Map); """ + sql """ CREATE FUNCTION udfii(Map) RETURNS INT PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.MapIntIntTest", + "type"="JAVA_UDF" + ); """ + + + qt_select_1 """ select m,udfii(m) from map_ii order by id; """ + + sql """ CREATE TABLE IF NOT EXISTS map_ss ( + `id` INT(11) NULL COMMENT "", + `m` Map NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ); """ + sql """ INSERT INTO map_ss VALUES(1, {"114":"514","1919":"810"}); """ + sql """ INSERT INTO map_ss VALUES(2, {"a":"bc","def":"g","hij":"k"}); """ + sql """ DROP FUNCTION IF EXISTS udfss(Map); """ + + sql """ CREATE FUNCTION udfss(Map) RETURNS STRING PROPERTIES ( + "file"="file://${jarPath}", + "symbol"="org.apache.doris.udf.MapStrStrTest", + "type"="JAVA_UDF" + ); """ + + qt_select_2 """ select m,udfss(m) from map_ss order by id; """ + } finally { + try_sql("DROP FUNCTION IF EXISTS udfii(Map);") + try_sql("DROP FUNCTION IF EXISTS udfss(Map);") + try_sql("DROP TABLE IF EXISTS map_ii") + try_sql("DROP TABLE IF EXISTS map_ss") + } +} From 65436af5b8e7dc5e51df6c64aae369507de8ca41 Mon Sep 17 00:00:00 2001 From: Mryange <2319153948@qq.com> Date: Mon, 24 Jul 2023 10:50:07 +0800 Subject: [PATCH 3/4] format --- .../org/apache/doris/udf/BaseExecutor.java | 129 +----------------- .../org/apache/doris/udf/UdfExecutor.java | 12 +- 2 files changed, 14 insertions(+), 127 deletions(-) diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java index 8cba4dc5100a60..abbddab4541f96 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java @@ -1203,135 +1203,20 @@ public Object[] convertArrayArg(int argIdx, boolean isNullable, int rowStart, in return argument; } - public Object[] convertMapKeyArg(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, + public Object[] convertMapArg(PrimitiveType type, int argIdx, boolean isNullable, int rowStart, int rowEnd, + long nullMapAddr, long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { Object[] argument = (Object[]) Array.newInstance(ArrayList.class, rowEnd - rowStart); for (int row = rowStart; row < rowEnd; ++row) { long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1)); long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row)); int currentRowNum = (int) (offsetEnd - offsetStart); - switch (argTypes[argIdx].getKeyType().getPrimitiveType()) { + switch (type) { case BOOLEAN: { - argument[row - rowStart] = UdfConvert - .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case TINYINT: { - argument[row - rowStart] = UdfConvert - .convertArrayTinyIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case SMALLINT: { - argument[row - rowStart] = UdfConvert - .convertArraySmallIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case INT: { - argument[row - rowStart] = UdfConvert - .convertArrayIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case BIGINT: { - argument[row - rowStart] = UdfConvert - .convertArrayBigIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case LARGEINT: { - argument[row - rowStart] = UdfConvert - .convertArrayLargeIntArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case FLOAT: { - argument[row - rowStart] = UdfConvert - .convertArrayFloatArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DOUBLE: { - argument[row - rowStart] = UdfConvert - .convertArrayDoubleArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case CHAR: - case VARCHAR: - case STRING: { - argument[row - rowStart] = UdfConvert - .convertArrayStringArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr, strOffsetAddr); - break; - } - case DATE: { - argument[row - rowStart] = UdfConvert - .convertArrayDateArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATETIME: { - argument[row - rowStart] = UdfConvert - .convertArrayDateTimeArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATEV2: { - argument[row - rowStart] = UdfConvert - .convertArrayDateV2Arg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); - break; - } - case DATETIMEV2: { - argument[row - rowStart] = UdfConvert - .convertArrayDateTimeV2Arg(row, currentRowNum, offsetStart, isNullable, - nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMALV2: - case DECIMAL128: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(argTypes[argIdx].getScale(), 16L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL32: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(argTypes[argIdx].getScale(), 4L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - case DECIMAL64: { - argument[row - rowStart] = UdfConvert - .convertArrayDecimalArg(argTypes[argIdx].getScale(), 8L, row, currentRowNum, - offsetStart, isNullable, nullMapAddr, nestedNullMapAddr, dataAddr); - break; - } - default: { - LOG.info("Not support: " + argTypes[argIdx]); - Preconditions.checkState(false, "Not support type " + argTypes[argIdx].toString()); - break; - } - } - } - return argument; - } - - public Object[] convertMapValueArg(int argIdx, boolean isNullable, int rowStart, int rowEnd, long nullMapAddr, - long offsetsAddr, long nestedNullMapAddr, long dataAddr, long strOffsetAddr) { - Object[] argument = (Object[]) Array.newInstance(ArrayList.class, rowEnd - rowStart); - for (int row = rowStart; row < rowEnd; ++row) { - long offsetStart = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row - 1)); - long offsetEnd = UdfUtils.UNSAFE.getLong(null, offsetsAddr + 8L * (row)); - int currentRowNum = (int) (offsetEnd - offsetStart); - switch (argTypes[argIdx].getValueType().getPrimitiveType()) { - case BOOLEAN: { - argument[row - rowStart] = UdfConvert - .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, - nestedNullMapAddr, dataAddr); + argument[row + - rowStart] = UdfConvert + .convertArrayBooleanArg(row, currentRowNum, offsetStart, isNullable, nullMapAddr, + nestedNullMapAddr, dataAddr); break; } case TINYINT: { diff --git a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java index 7af24809bbc2bf..34333ba1ebd6e1 100644 --- a/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java @@ -135,14 +135,14 @@ public Object[] convertArrayArguments(int argIdx, boolean isNullable, int numRow public Object[] convertMapArguments(int argIdx, boolean isNullable, int numRows, long nullMapAddr, long offsetsAddr, long keyNestedNullMapAddr, long keyDataAddr, long keyStrOffsetAddr, long valueNestedNullMapAddr, long valueDataAddr, long valueStrOffsetAddr) { - Object[] keyCol = convertMapKeyArg(argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, + PrimitiveType keyType = argTypes[argIdx].getKeyType().getPrimitiveType(); + PrimitiveType valueType = argTypes[argIdx].getValueType().getPrimitiveType(); + Object[] keyCol = convertMapArg(keyType, argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, keyNestedNullMapAddr, keyDataAddr, keyStrOffsetAddr); - Object[] valueCol = convertMapValueArg(argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, + Object[] valueCol = convertMapArg(valueType, argIdx, isNullable, 0, numRows, nullMapAddr, offsetsAddr, valueNestedNullMapAddr, valueDataAddr, valueStrOffsetAddr); - PrimitiveType keyType = argTypes[argIdx].getKeyType().getPrimitiveType(); - PrimitiveType valueType = argTypes[argIdx].getValueType().getPrimitiveType(); switch (keyType) { case BOOLEAN: { return new HashMapBuilder().get(keyCol, valueCol, valueType); @@ -640,7 +640,9 @@ public Object[] get(Object[] keyCol, Object[] valueCol) { for (int i = 0; i < keys.size(); i++) { T1 key = keys.get(i); T2 value = values.get(i); - hashMap.put(key, value); + if (!hashMap.containsKey(key)) { + hashMap.put(key, value); + } } retHashMap[colIdx] = hashMap; } From 51b06126adec8113ae6492783e5add98f98f08c5 Mon Sep 17 00:00:00 2001 From: Mryange <2319153948@qq.com> Date: Mon, 24 Jul 2023 11:55:55 +0800 Subject: [PATCH 4/4] remove enable --- regression-test/suites/javaudf_p0/test_javaudf_map.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/suites/javaudf_p0/test_javaudf_map.groovy b/regression-test/suites/javaudf_p0/test_javaudf_map.groovy index 8ca8f111f5a91b..4f7b89fd92ffe6 100644 --- a/regression-test/suites/javaudf_p0/test_javaudf_map.groovy +++ b/regression-test/suites/javaudf_p0/test_javaudf_map.groovy @@ -23,7 +23,6 @@ import java.nio.file.Paths suite("test_javaudf_map") { def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" - sql """ admin set frontend config("enable_map_type" = "true"); """ log.info("Jar path: ${jarPath}".toString()) try { try_sql("DROP FUNCTION IF EXISTS udfii(Map);")