Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions be/src/util/jni-util.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@
// under the License.

#include "util/jni-util.h"

#ifdef LIBJVM
#include <jni.h>
#include <jni_md.h>
#include <stdlib.h>

#include "gutil/once.h"
Expand Down Expand Up @@ -45,8 +47,10 @@ void FindOrCreateJavaVM() {
vm_args.nOptions = 1;
vm_args.ignoreUnrecognized = JNI_TRUE;

int res = JNI_CreateJavaVM(&g_vm, (void**)&env, &vm_args);
DCHECK_LT(res, 0) << "Failed tp create JVM, code= " << res;
jint res = JNI_CreateJavaVM(&g_vm, (void**)&env, &vm_args);
if (JNI_OK != res) {
DCHECK(false) << "Failed to create JVM, code= " << res;
}
} else {
CHECK_EQ(rv, 0) << "Could not find any created Java VM";
CHECK_EQ(num_vms, 1) << "No VMs returned";
Expand Down
22 changes: 13 additions & 9 deletions be/src/vec/functions/function_java_udf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,11 @@ JavaFunctionCall::JavaFunctionCall(const TFunction& fn, const DataTypes& argumen

Status JavaFunctionCall::prepare(FunctionContext* context,
FunctionContext::FunctionStateScope scope) {
DCHECK(executor_cl_ == NULL) << "Init() already called!";
JNIEnv* env;
JNIEnv* env = nullptr;
RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env));
if (env == NULL) return Status::InternalError("Failed to get/create JVM");
if (env == nullptr) {
return Status::InternalError("Failed to get/create JVM");
}
RETURN_IF_ERROR(JniUtil::GetGlobalClassRef(env, EXECUTOR_CLASS, &executor_cl_));
executor_ctor_id_ = env->GetMethodID(executor_cl_, "<init>", EXECUTOR_CTOR_SIGNATURE);
RETURN_ERROR_IF_EXC(env);
Expand Down Expand Up @@ -101,7 +102,7 @@ Status JavaFunctionCall::prepare(FunctionContext* context,
Status JavaFunctionCall::execute(FunctionContext* context, Block& block,
const ColumnNumbers& arguments, size_t result, size_t num_rows,
bool dry_run) {
JNIEnv* env;
JNIEnv* env = nullptr;
RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env));
JniContext* jni_ctx = reinterpret_cast<JniContext*>(
context->get_function_state(FunctionContext::THREAD_LOCAL));
Expand All @@ -124,12 +125,14 @@ Status JavaFunctionCall::execute(FunctionContext* context, Block& block,
} else {
jni_ctx->input_nulls_buffer_ptr.get()[arg_idx] = -1;
}
if (const ColumnString* str_col = check_and_get_column<ColumnString>(data_col.get())) {

if (data_col->is_column_string()) {
const ColumnString* str_col = assert_cast<const ColumnString*>(data_col.get());
jni_ctx->input_values_buffer_ptr.get()[arg_idx] =
reinterpret_cast<int64_t>(str_col->get_chars().data());
jni_ctx->input_offsets_ptrs.get()[arg_idx] =
reinterpret_cast<int64_t>(str_col->get_offsets().data());
} else if (data_col->is_numeric()) {
} else if (data_col->is_numeric() || data_col->is_column_decimal()) {
jni_ctx->input_values_buffer_ptr.get()[arg_idx] =
reinterpret_cast<int64_t>(data_col->get_raw_data().data);
} else {
Expand All @@ -151,7 +154,8 @@ Status JavaFunctionCall::execute(FunctionContext* context, Block& block,
*(jni_ctx->output_null_value) = reinterpret_cast<int64_t>(null_col->get_data().data());
#ifndef EVALUATE_JAVA_UDF
#define EVALUATE_JAVA_UDF \
if (const ColumnString* str_col = check_and_get_column<ColumnString>(data_col.get())) { \
if (data_col->is_column_string()) { \
const ColumnString* str_col = assert_cast<const ColumnString*>(data_col.get()); \
ColumnString::Chars& chars = const_cast<ColumnString::Chars&>(str_col->get_chars()); \
ColumnString::Offsets& offsets = \
const_cast<ColumnString::Offsets&>(str_col->get_offsets()); \
Expand All @@ -177,7 +181,7 @@ Status JavaFunctionCall::execute(FunctionContext* context, Block& block,
env->CallNonvirtualVoidMethodA(jni_ctx->executor, executor_cl_, executor_evaluate_id_, \
nullptr); \
} \
} else if (data_col->is_numeric()) { \
} else if (data_col->is_numeric() || data_col->is_column_decimal()) { \
data_col->reserve(num_rows); \
data_col->resize(num_rows); \
*(jni_ctx->output_value_buffer) = \
Expand Down Expand Up @@ -205,7 +209,7 @@ Status JavaFunctionCall::close(FunctionContext* context,
FunctionContext::FunctionStateScope scope) {
JniContext* jni_ctx = reinterpret_cast<JniContext*>(
context->get_function_state(FunctionContext::THREAD_LOCAL));
if (jni_ctx != NULL) {
if (jni_ctx != nullptr) {
delete jni_ctx;
context->set_function_state(FunctionContext::THREAD_LOCAL, nullptr);
}
Expand Down
19 changes: 18 additions & 1 deletion docs/en/ecosystem/udf/java-user-defined-function.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,23 @@ To use Java UDF, the main entry of UDF must be the `evaluate` function. This is

It is worth mentioning that this example is not only the Java UDF supported by Doris, but also the UDF supported by Hive, that's to say, for users, Hive UDF can be directly migrated to Doris.

#### Type correspondence

|UDF Type|Argument Type|
|----|---------|
|TinyInt|TinyIntVal|
|SmallInt|Short|
|Int|Integer|
|BigInt|Long|
|LargeInt|BigInteger|
|Float|Float|
|Double|Double|
|Date|LocalDate|
|Datetime|LocalDateTime|
|Char|String|
|Varchar|String|
|Decimal|BigDecimal|

## Create UDF

Currently, UDAF and UDTF are not supported.
Expand Down Expand Up @@ -85,6 +102,6 @@ Examples of Java UDF are provided in the `samples/doris-demo/java-udf-demo/` dir

## Unsupported Use Case
At present, Java UDF is still in the process of continuous development, so some features are **not completed**.
1. Complex data types (date, HLL, bitmap) are not supported.
1. Complex data types (HLL, bitmap) are not supported.
2. Memory management and statistics of JVM and Doris have not been unified.

20 changes: 19 additions & 1 deletion docs/zh-CN/ecosystem/udf/java-user-defined-function.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,24 @@ Java UDF 为用户提供UDF编写的Java接口,以方便用户使用Java语言
使用Java代码编写UDF,UDF的主入口必须为 `evaluate` 函数。这一点与Hive等其他引擎保持一致。在本示例中,我们编写了 `AddOne` UDF来完成对整型输入进行加一的操作。
值得一提的是,本例不只是Doris支持的Java UDF,同时还是Hive支持的UDF,也就是说,对于用户来讲,Hive UDF是可以直接迁移至Doris的。

#### 类型对应关系

|UDF Type|Argument Type|
|----|---------|
|TinyInt|TinyIntVal|
|SmallInt|Short|
|Int|Integer|
|BigInt|Long|
|LargeInt|BigInteger|
|Float|Float|
|Double|Double|
|Date|LocalDate|
|Datetime|LocalDateTime|
|Char|String|
|Varchar|String|
|Decimal|BigDecimal|


## 创建 UDF

目前暂不支持 UDAF 和 UDTF
Expand Down Expand Up @@ -84,6 +102,6 @@ UDF 的使用与普通的函数方式一致,唯一的区别在于,内置函

## 暂不支持的场景
当前Java UDF仍然处在持续的开发过程中,所以部分功能**尚不完善**。包括:
1. 不支持复杂数据类型(Date,HLL,Bitmap)
1. 不支持复杂数据类型(HLL,Bitmap)
2. 尚未统一JVM和Doris的内存管理以及统计信息

Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,15 @@
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.lang.reflect.Parameter;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -368,6 +372,10 @@ private void analyzeJavaUdf(String clazz) throws AnalysisException {
.put(PrimitiveType.CHAR, Sets.newHashSet(String.class))
.put(PrimitiveType.VARCHAR, Sets.newHashSet(String.class))
.put(PrimitiveType.STRING, Sets.newHashSet(String.class))
.put(PrimitiveType.DATE, Sets.newHashSet(LocalDate.class))
.put(PrimitiveType.DATETIME, Sets.newHashSet(LocalDateTime.class))
.put(PrimitiveType.LARGEINT, Sets.newHashSet(BigInteger.class))
.put(PrimitiveType.DECIMALV2, Sets.newHashSet(BigDecimal.class))
.build();

private void checkUdfType(Class clazz, Method method, Type expType, Class pType, String pname)
Expand Down
Loading