Support external users to contribute udf

be/CMakeLists.txt

@@ -498,7 +498,6 @@ add_subdirectory(${SRC_DIR}/testutil)
 #add_subdirectory(${SRC_DIR}/tools)
 add_subdirectory(${SRC_DIR}/udf)
 add_subdirectory(${SRC_DIR}/tools)
-add_subdirectory(${SRC_DIR}/udf_samples)
 add_subdirectory(${SRC_DIR}/util)
 add_subdirectory(${SRC_DIR}/plugin)
 

be/src/udf/udf.h

@@ -18,7 +18,7 @@
 #ifndef DORIS_BE_UDF_UDF_H
 #define DORIS_BE_UDF_UDF_H
 
-#include <boost/cstdint.hpp>
+#include <cstdint>
 #include <string.h>
 
 // This is the only Doris header required to develop UDFs and UDAs. This header

be/src/udf/udf_internal.h

@@ -18,7 +18,7 @@
 #ifndef DORIS_BE_UDF_UDF_INTERNAL_H
 #define DORIS_BE_UDF_UDF_INTERNAL_H
 
-#include <boost/cstdint.hpp>
+#include <cstdint>
 #include <map>
 #include <string>
 #include <string.h>

contrib/udf/CMakeLists.txt

@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 2.8.10)
+
+# set CMAKE_C_COMPILER, this must set before project command
+if (DEFINED ENV{DORIS_GCC_HOME})
+    set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc")
+    set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++")
+    set(GCC_HOME $ENV{DORIS_GCC_HOME})
+else()
+    message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set")
+endif()
+
+project(doris_udf)
+
+# set CMAKE_BUILD_TYPE
+if (NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE RELEASE)
+endif()
+
+string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+message(STATUS "Build type is ${CMAKE_BUILD_TYPE}")
+
+set(BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+set(ENV{DORIS_HOME} "${BASE_DIR}/../../")
+set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed/")
+set(BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
+set(SRC_DIR "${BASE_DIR}/src/")
+set(OUTPUT_DIR "${BASE_DIR}/output")
+
+# Check gcc
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.3.0")
+        message(FATAL_ERROR "Need GCC version at least 7.3.0")
+    endif()
+
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "7.3.0")
+        message(STATUS "GCC version is greater than 7.3.0, disable -Werror. Be careful with compile warnings.")
+    else()
+        #   -Werror: compile warnings should be errors when using the toolchain compiler.
+        set(CXX_GCC_FLAGS "${CXX_GCC_FLAGS} -Werror")
+    endif()
+elseif (NOT APPLE)
+    message(FATAL_ERROR "Compiler should be GNU")
+endif()
+
+# Just for clang-tidy:  -Wno-expansion-to-defined -Wno-deprecated-declaration
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -g -ggdb -std=c++11 -Wall -Werror -Wno-unused-variable -Wno-expansion-to-defined -Wno-deprecated-declarations -O3")
+message(STATUS "Compiler Flags: ${CMAKE_CXX_FLAGS}")
+
+# Include udf
+include_directories($ENV{DORIS_HOME}/output/udf/include)
+
+# Set all libraries
+add_library(udf STATIC IMPORTED)
+set_target_properties(udf PROPERTIES IMPORTED_LOCATION $ENV{DORIS_HOME}/output/udf/lib/libDorisUdf.a)
+
+# Add the subdirector of new UDF in here
+add_subdirectory(${SRC_DIR}/udf_samples)
+
+install(DIRECTORY DESTINATION ${OUTPUT_DIR})

contrib/udf/build_udf.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to compile UDF 
+# Usage:
+#    sh build-udf.sh             build udf without clean.
+#    sh build-udf.sh --clean     clean previous output and build.
+#
+##############################################################
+
+set -eo pipefail
+
+ROOT=`dirname "$0"`
+ROOT=`cd "$ROOT"; pwd`
+
+export UDF_HOME=${ROOT}
+export DORIS_HOME=$(cd ../..; printf %s "$PWD")
+echo ${DORIS_HOME}
+
+. ${DORIS_HOME}/env.sh
+
+PARALLEL=$[$(nproc)/4+1]
+
+# Check args
+usage() {
+  echo "
+Usage: $0 <options>
+  Optional options:
+     --clean            clean and build target
+
+  Eg.
+    $0                                 build UDF without clean
+    $0 --clean                         clean and build UDF
+  "
+  exit 1
+}
+
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -o 'h' \
+  -l 'clean' \
+  -l 'help' \
+  -- "$@")
+
+if [ $? != 0 ] ; then
+    usage
+fi
+
+eval set -- "$OPTS"
+
+BUILD_UDF=1
+CLEAN=0
+HELP=0
+if [ $# == 1 ] ; then
+    # default
+    CLEAN=0
+else
+    CLEAN=0
+    while true; do
+        case "$1" in
+            --clean) CLEAN=1 ; shift ;;
+            -h) HELP=1; shift ;;
+            --help) HELP=1; shift ;;
+            --) shift ;  break ;;
+            *) ehco "Internal error" ; exit 1 ;;
+        esac
+    done
+fi
+
+if [[ ${HELP} -eq 1 ]]; then
+    usage
+    exit
+fi
+
+echo "Get params:
+    CLEAN       -- $CLEAN
+"
+
+cd ${UDF_HOME}
+# Clean and build UDF
+if [ ${BUILD_UDF} -eq 1 ] ; then
+    CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release}
+    echo "Build UDF: ${CMAKE_BUILD_TYPE}"
+    CMAKE_BUILD_DIR=${UDF_HOME}/build_${CMAKE_BUILD_TYPE}
+    if [ ${CLEAN} -eq 1 ]; then
+        rm -rf $CMAKE_BUILD_DIR
+        rm -rf ${UDF_HOME}/output/
+    fi
+    mkdir -p ${CMAKE_BUILD_DIR}
+    cd ${CMAKE_BUILD_DIR}
+    ${CMAKE_CMD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ../
+    make -j${PARALLEL} VERBOSE=1
+    make install
+    cd ${UDF_HOME}
+fi
+
+# Clean and prepare output dir
+DORIS_OUTPUT=${DORIS_HOME}/output/
+mkdir -p ${DORIS_OUTPUT}
+
+#Copy UDF
+if [ ${BUILD_UDF} -eq 1 ]; then
+    install -d ${DORIS_OUTPUT}/contrib/udf/lib
+    for dir in $(ls ${CMAKE_BUILD_DIR}/src)
+    do
+      mkdir -p ${DORIS_OUTPUT}/contrib/udf/lib/$dir
+      cp -r -p ${CMAKE_BUILD_DIR}/src/$dir/*.so ${DORIS_OUTPUT}/contrib/udf/lib/$dir/
+    done
+fi
+
+echo "***************************************"
+echo "Successfully build Doris UDF"
+echo "***************************************"
+
+if [[ ! -z ${DORIS_POST_BUILD_HOOK} ]]; then
+    eval ${DORIS_POST_BUILD_HOOK}
+fi
+
+exit 0

be/src/udf_samples/uda_sample.cpp → contrib/udf/src/udf_samples/uda_sample.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "udf_samples/udf_sample.h"
+#include "udf_sample.h"
 
 namespace doris_udf {
 

be/src/udf_samples/uda_sample.h → contrib/udf/src/udf_samples/uda_sample.h

@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include "udf/udf.h"
+#include "udf.h"
 
 namespace doris_udf {
 

be/src/udf_samples/udf_sample.cpp → contrib/udf/src/udf_samples/udf_sample.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "udf_samples/udf_sample.h"
+#include "udf_sample.h"
 
 namespace doris_udf {
 

be/src/udf_samples/udf_sample.h → contrib/udf/src/udf_samples/udf_sample.h

@@ -17,7 +17,7 @@
 
 #pragma once
 
-#include "udf/udf.h"
+#include "udf.h"
 
 namespace doris_udf {
 

docs/.vuepress/sidebar/en.js

@@ -124,9 +124,21 @@ module.exports = [
       "audit-plugin",
       "doris-on-es",
       "plugin-development-manual",
-      "user-defined-function",
       "spark-doris-connector",
       "logstash",
+      {
+        title: "UDF",
+        directoryPath: "udf/",
+        children: [
+          "user-defined-function",
+          "contribute-udf",
+          {
+            title: "Users contribute UDF",
+            directoryPath: "contrib/",
+            children:[],       
+          },          
+        ],
+      },
     ],
   },
   {

docs/.vuepress/sidebar/zh-CN.js

@@ -135,9 +135,21 @@ module.exports = [
       "audit-plugin",
       "doris-on-es",
       "plugin-development-manual",
-      "user-defined-function",
       "spark-doris-connector",
       "logstash",
+      {
+        title: "UDF",
+        directoryPath: "udf/",
+        children: [
+          "user-defined-function",
+          "contribute-udf",
+          {
+            title: "用户贡献的 UDF",
+            directoryPath: "contrib/",
+            children:[],       
+          },          
+        ],
+      },
     ],
   },
   {