diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 2c3f5b74083b19..19606dedcbd8e7 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -18,13 +18,6 @@ cmake_minimum_required(VERSION 3.19.2) # set CMAKE_C_COMPILER, this must set before project command -if (DEFINED ENV{DORIS_GCC_HOME}) - set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc") - set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++") - set(GCC_HOME $ENV{DORIS_GCC_HOME}) -else() - message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set") -endif() project(doris CXX C) @@ -89,16 +82,6 @@ endif() message(STATUS "make test: ${MAKE_TEST}") option(WITH_MYSQL "Support access MySQL" ON) -# Check gcc -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.3.0") - message(FATAL_ERROR "Need GCC version at least 7.3.0") - endif() - message(STATUS "GCC version is greater than 7.3.0, disable -Werror. Be careful with compile warnings.") -elseif (NOT APPLE) - message(FATAL_ERROR "Compiler should be GNU") -endif() - set(PIC_LIB_PATH "${THIRDPARTY_DIR}") if(PIC_LIB_PATH) message(STATUS "defined PIC_LIB_PATH") diff --git a/be/src/exec/olap_utils.h b/be/src/exec/olap_utils.h index 129bd2c6eb0624..bc43df68cf238c 100644 --- a/be/src/exec/olap_utils.h +++ b/be/src/exec/olap_utils.h @@ -75,6 +75,8 @@ inline CompareLargeFunc get_compare_func(PrimitiveType type) { default: DCHECK(false) << "Unsupported Compare type"; + // Clang will report error if not all path have return value + return nullptr; } } diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp index 7e3cf3065bb137..d9cf3176e3d578 100644 --- a/be/src/olap/column_vector.cpp +++ b/be/src/olap/column_vector.cpp @@ -164,9 +164,6 @@ ScalarColumnVectorBatch::ScalarColumnVectorBatch(const TypeInfo* typ bool is_nullable) : ColumnVectorBatch(type_info, is_nullable), _data(0) {} -template -ScalarColumnVectorBatch::~ScalarColumnVectorBatch() = default; - template Status ScalarColumnVectorBatch::resize(size_t new_cap) { if (capacity() < new_cap) { // before first init, _capacity is 0. @@ -222,35 +219,4 @@ void ArrayColumnVectorBatch::prepare_for_read(size_t start_idx, size_t size, boo } } -template -DataBuffer::DataBuffer(size_t new_size) : buf(nullptr), current_size(0), current_capacity(0) { - resize(new_size); -} - -template -DataBuffer::~DataBuffer() { - for (uint64_t i = current_size; i > 0; --i) { - (buf + i - 1)->~T(); - } - if (buf) { - std::free(buf); - } -} - -template -void DataBuffer::resize(size_t new_size) { - if (new_size > current_capacity || !buf) { - if (buf) { - T* buf_old = buf; - buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); - memcpy(buf, buf_old, sizeof(T) * current_size); - std::free(buf_old); - } else { - buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); - } - current_capacity = new_size; - } - current_size = new_size; -} - } // namespace doris diff --git a/be/src/olap/column_vector.h b/be/src/olap/column_vector.h index ca5b6e1c4daa4d..4ab97130deb3bb 100644 --- a/be/src/olap/column_vector.h +++ b/be/src/olap/column_vector.h @@ -51,6 +51,37 @@ class DataBuffer { void resize(size_t _size); }; +template +DataBuffer::DataBuffer(size_t new_size) : buf(nullptr), current_size(0), current_capacity(0) { + resize(new_size); +} + +template +DataBuffer::~DataBuffer() { + for (uint64_t i = current_size; i > 0; --i) { + (buf + i - 1)->~T(); + } + if (buf) { + std::free(buf); + } +} + +template +void DataBuffer::resize(size_t new_size) { + if (new_size > current_capacity || !buf) { + if (buf) { + T* buf_old = buf; + buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); + memcpy(buf, buf_old, sizeof(T) * current_size); + std::free(buf_old); + } else { + buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); + } + current_capacity = new_size; + } + current_size = new_size; +} + template class DataBuffer; template class DataBuffer; template class DataBuffer; @@ -161,6 +192,9 @@ class ScalarColumnVectorBatch : public ColumnVectorBatch { DataBuffer _data; }; +template +ScalarColumnVectorBatch::~ScalarColumnVectorBatch() = default; + // util class for read array's null signs. class ArrayNullColumnVectorBatch : public ColumnVectorBatch { public: diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 4dddf08d889a36..ec59a92d4bdae6 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -420,7 +420,7 @@ struct CppTypeTraits { template <> struct CppTypeTraits { using CppType = int128_t; - using UnsignedCppType = unsigned int128_t; + using UnsignedCppType = uint128_t; }; template <> struct CppTypeTraits { diff --git a/be/src/runtime/dpp_sink_internal.cpp b/be/src/runtime/dpp_sink_internal.cpp index ad172020fbbeb7..72cf22ed4ced4c 100644 --- a/be/src/runtime/dpp_sink_internal.cpp +++ b/be/src/runtime/dpp_sink_internal.cpp @@ -31,11 +31,12 @@ namespace doris { PartRangeKey PartRangeKey::_s_pos_infinite(1); PartRangeKey PartRangeKey::_s_neg_infinite(-1); +// Clang will check constructor method not exist PartRange PartRange::_s_all_range = { - ._start_key = PartRangeKey::neg_infinite(), - ._end_key = PartRangeKey::pos_infinite(), - ._include_start_key = true, - ._include_end_key = true, + PartRangeKey::neg_infinite(), + PartRangeKey::pos_infinite(), + true, + true }; RollupSchema::RollupSchema() {} diff --git a/be/src/runtime/raw_value_ir.cpp b/be/src/runtime/raw_value_ir.cpp index 28375759150140..d6fef33f437b02 100644 --- a/be/src/runtime/raw_value_ir.cpp +++ b/be/src/runtime/raw_value_ir.cpp @@ -16,6 +16,7 @@ // under the License. #include "runtime/raw_value.h" +#include "runtime/string_value.hpp" #include "util/types.h" namespace doris { diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 81fb10bbd7ec6a..7d0c64294c979e 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1734,10 +1734,6 @@ class BitmapValue { // } class BitmapValueIterator { public: - BitmapValueIterator() - : _bitmap(BitmapValue()) { - } - BitmapValueIterator(const BitmapValue& bitmap, bool end = false) : _bitmap(bitmap), _end(end) { diff --git a/build.sh b/build.sh index a5ccaae5e8a75b..8c3dd81c8ee190 100755 --- a/build.sh +++ b/build.sh @@ -38,10 +38,25 @@ set -eo pipefail ROOT=`dirname "$0"` ROOT=`cd "$ROOT"; pwd` + export DORIS_HOME=${ROOT} +#export CC="/data/common/clang-11/bin/clang-wrapper" +#export CXX="/data/common/clang-11/bin/clang++-wrapper" + +#export CC="/usr/local/gcc-10.1.0/bin/gcc" +#export CXX="/usr/local/gcc-10.1.0/bin/g++" + . ${DORIS_HOME}/env.sh +if [[ -z ${CC} ]]; then + if [[ -z ${DORIS_GCC_HOME} ]]; then + DORIS_GCC_HOME=$(dirname $(which gcc))/.. + fi + export CC="${DORIS_GCC_HOME}/bin/gcc" + export CXX="${DORIS_GCC_HOME}/bin/g++" +fi + # Check args usage() { echo " diff --git a/contrib/udf/CMakeLists.txt b/contrib/udf/CMakeLists.txt index 30ce621e0435d7..45a41ab4e920d6 100644 --- a/contrib/udf/CMakeLists.txt +++ b/contrib/udf/CMakeLists.txt @@ -18,13 +18,6 @@ cmake_minimum_required(VERSION 3.19.2) # set CMAKE_C_COMPILER, this must set before project command -if (DEFINED ENV{DORIS_GCC_HOME}) - set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc") - set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++") - set(GCC_HOME $ENV{DORIS_GCC_HOME}) -else() - message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set") -endif() project(doris_udf) diff --git a/env.sh b/env.sh index bf64801532fe03..37fef4ab465c20 100755 --- a/env.sh +++ b/env.sh @@ -51,30 +51,25 @@ if ! ${PYTHON} --version; then fi fi -# set GCC HOME -if [[ -z ${DORIS_GCC_HOME} ]]; then - export DORIS_GCC_HOME=$(dirname $(which gcc))/.. -fi - -gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion) -required_ver="7.3.0" -if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then - echo "Error: GCC version (${gcc_ver}) must be greater than or equal to ${required_ver}" +# register keyword is forbidden to use in C++17 +# the C++ code generated by flex that remove register keyword after version 2.6.0 +# so we need check flex version here to avoid compilation failed +flex_ver=$(flex --version | awk '{print $2}') +required_ver="2.6.0" +if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then + echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}" exit 1 fi -# find binutils -if test -x ${DORIS_GCC_HOME}/bin/ld; then - export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/ +# check java home +if [ -z "$JAVA_HOME" ]; then + export JAVACMD=$(which java) + JAVAP=$(which javap) else - export DORIS_BIN_UTILS=/usr/bin/ + export JAVA="${JAVA_HOME}/bin/java" + JAVAP="${JAVA_HOME}/bin/javap" fi -# export CLANG COMPATIBLE FLAGS -export CLANG_COMPATIBLE_FLAGS=$(echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 | - grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ') - - # if is called from build-thirdparty.sh, no need to check these tools if test -z "${BUILD_THIRDPARTY_WIP}"; then # register keyword is forbidden to use in C++17 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index cedec4934314ce..a80b539d52c164 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -105,16 +105,6 @@ ${TP_DIR}/download-thirdparty.sh export LD_LIBRARY_PATH=$TP_DIR/installed/lib:$LD_LIBRARY_PATH -# set COMPILER -if [[ ! -z ${DORIS_GCC_HOME} ]]; then - export CC=${DORIS_GCC_HOME}/bin/gcc - export CPP=${DORIS_GCC_HOME}/bin/cpp - export CXX=${DORIS_GCC_HOME}/bin/g++ -else - echo "DORIS_GCC_HOME environment variable is not set" - exit 1 -fi - # prepare installed prefix mkdir -p ${TP_DIR}/installed/lib64 pushd ${TP_DIR}/installed/ @@ -184,6 +174,8 @@ check_if_source_exist() { echo "$TP_SOURCE_DIR/$1 does not exist." exit 1 fi + # Remove the build dir to clean build env + rm -rf $TP_SOURCE_DIR/$1/$BUILD_DIR echo "===== begin build $1" } @@ -208,7 +200,7 @@ build_libevent() { CFLAGS="-std=c99 -fPIC -D_BSD_SOURCE -fno-omit-frame-pointer -g -ggdb -O2 -I${TP_INCLUDE_DIR}" \ CPPLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ - ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DEVENT__DISABLE_TESTS=ON \ + ${CMAKE_CMD} -G "${GENERATOR}" -DEVENT__LIBRARY_TYPE=STATIC -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DEVENT__DISABLE_TESTS=ON \ -DEVENT__DISABLE_OPENSSL=ON -DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON .. ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } @@ -228,7 +220,7 @@ build_openssl() { LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ LIBDIR="lib" \ - ./Configure --prefix=$TP_INSTALL_DIR --with-rand-seed=devrandom -zlib -shared ${OPENSSL_PLATFORM} + ./Configure --prefix=$TP_INSTALL_DIR --with-rand-seed=devrandom -shared ${OPENSSL_PLATFORM} make -j $PARALLEL && make install_sw # NOTE(zc): remove this dynamic library files to make libcurl static link. # If I don't remove this files, I don't known how to make libcurl link static library @@ -354,10 +346,6 @@ build_gperftools() { CPPFLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ - LD_LIBRARY_PATH="${TP_LIB_DIR}" \ - CFLAGS="-fPIC" \ - LDFLAGS="-L${TP_LIB_DIR}" \ - LD_LIBRARY_PATH="${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR/gperftools --disable-shared --enable-static --disable-libunwind --with-pic --enable-frame-pointers make -j $PARALLEL && make install @@ -517,8 +505,10 @@ build_rocksdb() { cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE - CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I ${TP_INCLUDE_DIR}/lz4" CXXFLAGS="-fPIC -Wno-deprecated-copy -Wno-stringop-truncation -Wno-pessimizing-move" LDFLAGS="-static-libstdc++ -static-libgcc" \ - PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib + CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I ${TP_INCLUDE_DIR}/lz4" \ + CXXFLAGS="-fPIC -w -Wno-deprecated-copy -Wno-shadow -Wno-range-loop-construct -Wno-dangling-gsl -Wno-pessimizing-move" \ + LDFLAGS="-static-libstdc++ -static-libgcc" \ + PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib cp librocksdb.a ../../installed/lib/librocksdb.a cp -r include/rocksdb ../../installed/include/ } @@ -566,7 +556,7 @@ build_flatbuffers() { cd $TP_SOURCE_DIR/$FLATBUFFERS_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR rm -rf CMakeCache.txt CMakeFiles/ - CXXFLAGS="-fPIC -Wno-class-memaccess" \ + CXXFLAGS="-fPIC " \ LDFLAGS="-static-libstdc++ -static-libgcc" \ ${CMAKE_CMD} -G "${GENERATOR}" .. ${BUILD_SYSTEM} -j $PARALLEL @@ -643,8 +633,28 @@ build_s2() { ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } -# bitshuffle build_bitshuffle() { + compiler_cmd=`which ${CC}` + compiler_tool_home=`dirname ${compiler_cmd}` + compiler_tool_home=`cd "$compiler_tool_home/.."; pwd` + + # find binutils + if test -x ${compiler_tool_home}/bin/llvm-objcopy; then + export LD_CMD=${compiler_tool_home}/bin/ld.lld + export NM_CMD=${compiler_tool_home}/bin/llvm-nm + export AR_CMD=${compiler_tool_home}/bin/llvm-ar + export OBJCOPY_CMD=${compiler_tool_home}/bin/llvm-objcopy + elif test -x ${compiler_tool_home}/bin/ld; then + export LD_CMD=${compiler_tool_home}/bin/ld + export NM_CMD=${compiler_tool_home}/bin/nm + export AR_CMD=${compiler_tool_home}/bin/ar + export OBJCOPY_CMD=${compiler_tool_home}/bin/objcopy + else + export LD_CMD=/usr/bin/ld + export NM_CMD=/usr/bin/nm + export AR_CMD=/usr/bin/ar + export OBJCOPY_CMD=/usr/bin/objcopy + fi check_if_source_exist $BITSHUFFLE_SOURCE cd $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE PREFIX=$TP_INSTALL_DIR @@ -668,26 +678,26 @@ build_bitshuffle() { fi tmp_obj=bitshuffle_${arch}_tmp.o dst_obj=bitshuffle_${arch}.o - ${CC:-$DORIS_GCC_HOME/bin/gcc} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \ + ${CC} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \ "src/bitshuffle_core.c" \ "src/bitshuffle.c" \ "src/iochain.c" # Merge the object files together to produce a combined .o file. - $DORIS_BIN_UTILS/ld -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o + ${LD_CMD} -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o # For the AVX2 symbols, suffix them. if [ "$arch" == "avx2" ]; then # Create a mapping file with ' ' on each line. - $DORIS_BIN_UTILS/nm --defined-only --extern-only $tmp_obj | while read addr type sym ; do + ${NM_CMD} --defined-only --extern-only $tmp_obj | while read addr type sym ; do echo ${sym} ${sym}_${arch} done > renames.txt - $DORIS_BIN_UTILS/objcopy --redefine-syms=renames.txt $tmp_obj $dst_obj + ${OBJCOPY_CMD} --redefine-syms=renames.txt $tmp_obj $dst_obj else mv $tmp_obj $dst_obj fi to_link="$to_link $dst_obj" done rm -f libbitshuffle.a - $DORIS_BIN_UTILS/ar rs libbitshuffle.a $to_link + ${AR_CMD} rs libbitshuffle.a $to_link mkdir -p $PREFIX/include/bitshuffle cp libbitshuffle.a $PREFIX/lib/ cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle.h $PREFIX/include/bitshuffle/bitshuffle.h @@ -744,7 +754,7 @@ build_orc() { cd $TP_SOURCE_DIR/$ORC_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR rm -rf CMakeCache.txt CMakeFiles/ - CXXFLAGS="-O3 -Wno-array-bounds" \ + CXXFLAGS="-O3 -Wno-array-bounds -Wno-suggest-destructor-override -Wno-suggest-override" \ ${CMAKE_CMD} -G "${GENERATOR}" ../ -DBUILD_JAVA=OFF \ -DPROTOBUF_HOME=$TP_INSTALL_DIR \ -DSNAPPY_HOME=$TP_INSTALL_DIR \ @@ -901,6 +911,7 @@ build_hdfs3() { check_if_source_exist $HDFS3_SOURCE cd $TP_SOURCE_DIR/$HDFS3_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR + LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \ ../bootstrap --dependency=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR make -j $PARALLEL && make install }