From f46ede6af00b1bd72c51c4aa26ba4dadf75724b8 Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Tue, 21 Dec 2021 14:08:33 +0800 Subject: [PATCH 1/8] Support build with clang --- be/CMakeLists.txt | 17 -------------- be/src/exec/olap_utils.h | 2 ++ be/src/olap/column_vector.cpp | 34 ---------------------------- be/src/olap/column_vector.h | 34 ++++++++++++++++++++++++++++ be/src/olap/types.h | 2 +- be/src/runtime/dpp_sink_internal.cpp | 9 ++++---- be/src/runtime/raw_value_ir.cpp | 1 + be/src/util/bitmap_value.h | 2 +- build.sh | 7 ++++++ contrib/udf/CMakeLists.txt | 7 ------ 10 files changed, 51 insertions(+), 64 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 2c3f5b74083b19..19606dedcbd8e7 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -18,13 +18,6 @@ cmake_minimum_required(VERSION 3.19.2) # set CMAKE_C_COMPILER, this must set before project command -if (DEFINED ENV{DORIS_GCC_HOME}) - set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc") - set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++") - set(GCC_HOME $ENV{DORIS_GCC_HOME}) -else() - message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set") -endif() project(doris CXX C) @@ -89,16 +82,6 @@ endif() message(STATUS "make test: ${MAKE_TEST}") option(WITH_MYSQL "Support access MySQL" ON) -# Check gcc -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "7.3.0") - message(FATAL_ERROR "Need GCC version at least 7.3.0") - endif() - message(STATUS "GCC version is greater than 7.3.0, disable -Werror. Be careful with compile warnings.") -elseif (NOT APPLE) - message(FATAL_ERROR "Compiler should be GNU") -endif() - set(PIC_LIB_PATH "${THIRDPARTY_DIR}") if(PIC_LIB_PATH) message(STATUS "defined PIC_LIB_PATH") diff --git a/be/src/exec/olap_utils.h b/be/src/exec/olap_utils.h index 129bd2c6eb0624..bc43df68cf238c 100644 --- a/be/src/exec/olap_utils.h +++ b/be/src/exec/olap_utils.h @@ -75,6 +75,8 @@ inline CompareLargeFunc get_compare_func(PrimitiveType type) { default: DCHECK(false) << "Unsupported Compare type"; + // Clang will report error if not all path have return value + return nullptr; } } diff --git a/be/src/olap/column_vector.cpp b/be/src/olap/column_vector.cpp index 7e3cf3065bb137..d9cf3176e3d578 100644 --- a/be/src/olap/column_vector.cpp +++ b/be/src/olap/column_vector.cpp @@ -164,9 +164,6 @@ ScalarColumnVectorBatch::ScalarColumnVectorBatch(const TypeInfo* typ bool is_nullable) : ColumnVectorBatch(type_info, is_nullable), _data(0) {} -template -ScalarColumnVectorBatch::~ScalarColumnVectorBatch() = default; - template Status ScalarColumnVectorBatch::resize(size_t new_cap) { if (capacity() < new_cap) { // before first init, _capacity is 0. @@ -222,35 +219,4 @@ void ArrayColumnVectorBatch::prepare_for_read(size_t start_idx, size_t size, boo } } -template -DataBuffer::DataBuffer(size_t new_size) : buf(nullptr), current_size(0), current_capacity(0) { - resize(new_size); -} - -template -DataBuffer::~DataBuffer() { - for (uint64_t i = current_size; i > 0; --i) { - (buf + i - 1)->~T(); - } - if (buf) { - std::free(buf); - } -} - -template -void DataBuffer::resize(size_t new_size) { - if (new_size > current_capacity || !buf) { - if (buf) { - T* buf_old = buf; - buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); - memcpy(buf, buf_old, sizeof(T) * current_size); - std::free(buf_old); - } else { - buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); - } - current_capacity = new_size; - } - current_size = new_size; -} - } // namespace doris diff --git a/be/src/olap/column_vector.h b/be/src/olap/column_vector.h index ca5b6e1c4daa4d..4ab97130deb3bb 100644 --- a/be/src/olap/column_vector.h +++ b/be/src/olap/column_vector.h @@ -51,6 +51,37 @@ class DataBuffer { void resize(size_t _size); }; +template +DataBuffer::DataBuffer(size_t new_size) : buf(nullptr), current_size(0), current_capacity(0) { + resize(new_size); +} + +template +DataBuffer::~DataBuffer() { + for (uint64_t i = current_size; i > 0; --i) { + (buf + i - 1)->~T(); + } + if (buf) { + std::free(buf); + } +} + +template +void DataBuffer::resize(size_t new_size) { + if (new_size > current_capacity || !buf) { + if (buf) { + T* buf_old = buf; + buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); + memcpy(buf, buf_old, sizeof(T) * current_size); + std::free(buf_old); + } else { + buf = reinterpret_cast(std::malloc(sizeof(T) * new_size)); + } + current_capacity = new_size; + } + current_size = new_size; +} + template class DataBuffer; template class DataBuffer; template class DataBuffer; @@ -161,6 +192,9 @@ class ScalarColumnVectorBatch : public ColumnVectorBatch { DataBuffer _data; }; +template +ScalarColumnVectorBatch::~ScalarColumnVectorBatch() = default; + // util class for read array's null signs. class ArrayNullColumnVectorBatch : public ColumnVectorBatch { public: diff --git a/be/src/olap/types.h b/be/src/olap/types.h index 4dddf08d889a36..ec59a92d4bdae6 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -420,7 +420,7 @@ struct CppTypeTraits { template <> struct CppTypeTraits { using CppType = int128_t; - using UnsignedCppType = unsigned int128_t; + using UnsignedCppType = uint128_t; }; template <> struct CppTypeTraits { diff --git a/be/src/runtime/dpp_sink_internal.cpp b/be/src/runtime/dpp_sink_internal.cpp index ad172020fbbeb7..72cf22ed4ced4c 100644 --- a/be/src/runtime/dpp_sink_internal.cpp +++ b/be/src/runtime/dpp_sink_internal.cpp @@ -31,11 +31,12 @@ namespace doris { PartRangeKey PartRangeKey::_s_pos_infinite(1); PartRangeKey PartRangeKey::_s_neg_infinite(-1); +// Clang will check constructor method not exist PartRange PartRange::_s_all_range = { - ._start_key = PartRangeKey::neg_infinite(), - ._end_key = PartRangeKey::pos_infinite(), - ._include_start_key = true, - ._include_end_key = true, + PartRangeKey::neg_infinite(), + PartRangeKey::pos_infinite(), + true, + true }; RollupSchema::RollupSchema() {} diff --git a/be/src/runtime/raw_value_ir.cpp b/be/src/runtime/raw_value_ir.cpp index 28375759150140..d6fef33f437b02 100644 --- a/be/src/runtime/raw_value_ir.cpp +++ b/be/src/runtime/raw_value_ir.cpp @@ -16,6 +16,7 @@ // under the License. #include "runtime/raw_value.h" +#include "runtime/string_value.hpp" #include "util/types.h" namespace doris { diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 81fb10bbd7ec6a..ecd55a9128cad6 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1832,7 +1832,7 @@ class BitmapValueIterator { } private: - const BitmapValue& _bitmap; + const BitmapValue _bitmap; detail::Roaring64MapSetBitForwardIterator* _iter = nullptr; uint64_t _sv = 0; bool _end = false; diff --git a/build.sh b/build.sh index a5ccaae5e8a75b..b2e889591a71ce 100755 --- a/build.sh +++ b/build.sh @@ -38,6 +38,7 @@ set -eo pipefail ROOT=`dirname "$0"` ROOT=`cd "$ROOT"; pwd` + export DORIS_HOME=${ROOT} . ${DORIS_HOME}/env.sh @@ -195,6 +196,12 @@ make # Clean and build Backend if [ ${BUILD_BE} -eq 1 ] ; then + # Has to set CC and CXX here + # --gcc-toolchain=/usr/local/gcc-10.1.0 + # Could set use clang here to build BE with clang + #export CC="/data/common/clang-11/bin/clang --gcc-toolchain=${DORIS_GCC_HOME}" + #export CXX="/data/common/clang-11/bin/clang++ --gcc-toolchain=${DORIS_GCC_HOME}" + CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release} echo "Build Backend: ${CMAKE_BUILD_TYPE}" CMAKE_BUILD_DIR=${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE} diff --git a/contrib/udf/CMakeLists.txt b/contrib/udf/CMakeLists.txt index 30ce621e0435d7..45a41ab4e920d6 100644 --- a/contrib/udf/CMakeLists.txt +++ b/contrib/udf/CMakeLists.txt @@ -18,13 +18,6 @@ cmake_minimum_required(VERSION 3.19.2) # set CMAKE_C_COMPILER, this must set before project command -if (DEFINED ENV{DORIS_GCC_HOME}) - set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc") - set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++") - set(GCC_HOME $ENV{DORIS_GCC_HOME}) -else() - message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set") -endif() project(doris_udf) From 4b989b1b8d9614dfe7991029104ee77766c276d8 Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Tue, 21 Dec 2021 14:33:20 +0800 Subject: [PATCH 2/8] build with clang --- be/src/util/bitmap_value.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index ecd55a9128cad6..eea08edd8e187c 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1735,7 +1735,7 @@ class BitmapValue { class BitmapValueIterator { public: BitmapValueIterator() - : _bitmap(BitmapValue()) { + : _bitmap() { } BitmapValueIterator(const BitmapValue& bitmap, bool end = false) From 3d7947667b03f2c7742a726202b227235c87d494 Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Tue, 21 Dec 2021 14:38:18 +0800 Subject: [PATCH 3/8] Remove unused bitmapvalueiterator constructor --- be/src/util/bitmap_value.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index eea08edd8e187c..772cf596f4aad8 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1734,10 +1734,6 @@ class BitmapValue { // } class BitmapValueIterator { public: - BitmapValueIterator() - : _bitmap() { - } - BitmapValueIterator(const BitmapValue& bitmap, bool end = false) : _bitmap(bitmap), _end(end) { From cb0b9b8c6137656ead14141932a93587d8c7cfa8 Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Tue, 21 Dec 2021 14:40:32 +0800 Subject: [PATCH 4/8] Remove unused bitmapvalueiterator constructor --- be/src/util/bitmap_value.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 772cf596f4aad8..7d0c64294c979e 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1828,7 +1828,7 @@ class BitmapValueIterator { } private: - const BitmapValue _bitmap; + const BitmapValue& _bitmap; detail::Roaring64MapSetBitForwardIterator* _iter = nullptr; uint64_t _sv = 0; bool _end = false; From 0255e8d8407e556f3919f62f31bca5492fc51f63 Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Wed, 22 Dec 2021 14:45:12 +0800 Subject: [PATCH 5/8] Could not build bitshuffle because it changed synoms --- build.sh | 7 ++++ env.sh | 21 +++++++++++ thirdparty/build-thirdparty.sh | 64 ++++------------------------------ 3 files changed, 35 insertions(+), 57 deletions(-) diff --git a/build.sh b/build.sh index b2e889591a71ce..5c6a7f93ee859e 100755 --- a/build.sh +++ b/build.sh @@ -41,6 +41,13 @@ ROOT=`cd "$ROOT"; pwd` export DORIS_HOME=${ROOT} +#export CC="/data/common/clang-11/bin/clang --gcc-toolchain=/usr/local/gcc-10.1.0" +#export CXX="/data/common/clang-11/bin/clang++ --gcc-toolchain=/usr/local/gcc-10.1.0" + +export CC="/usr/local/gcc-10.1.0/bin/gcc" +export CXX="/usr/local/gcc-10.1.0/bin/g++" + + . ${DORIS_HOME}/env.sh # Check args diff --git a/env.sh b/env.sh index bf64801532fe03..da74e8ac9da833 100755 --- a/env.sh +++ b/env.sh @@ -63,6 +63,18 @@ if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$r exit 1 fi +# register keyword is forbidden to use in C++17 +# the C++ code generated by flex that remove register keyword after version 2.6.0 +# so we need check flex version here to avoid compilation failed +flex_ver=$(flex --version | awk '{print $2}') +required_ver="2.6.0" +if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then + echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}" + exit 1 +fi + +<<<<<<< HEAD +>>>>>>> Could not build bitshuffle because it changed synoms # find binutils if test -x ${DORIS_GCC_HOME}/bin/ld; then export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/ @@ -74,6 +86,15 @@ fi export CLANG_COMPATIBLE_FLAGS=$(echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 | grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ') +# check java home +if [ -z "$JAVA_HOME" ]; then + export JAVACMD=$(which java) + JAVAP=$(which javap) +else + export JAVA="${JAVA_HOME}/bin/java" + JAVAP="${JAVA_HOME}/bin/javap" +fi +>>>>>>> Could not build bitshuffle because it changed synoms # if is called from build-thirdparty.sh, no need to check these tools if test -z "${BUILD_THIRDPARTY_WIP}"; then diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index cedec4934314ce..0dd7b2ddd7e4f1 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -105,16 +105,6 @@ ${TP_DIR}/download-thirdparty.sh export LD_LIBRARY_PATH=$TP_DIR/installed/lib:$LD_LIBRARY_PATH -# set COMPILER -if [[ ! -z ${DORIS_GCC_HOME} ]]; then - export CC=${DORIS_GCC_HOME}/bin/gcc - export CPP=${DORIS_GCC_HOME}/bin/cpp - export CXX=${DORIS_GCC_HOME}/bin/g++ -else - echo "DORIS_GCC_HOME environment variable is not set" - exit 1 -fi - # prepare installed prefix mkdir -p ${TP_DIR}/installed/lib64 pushd ${TP_DIR}/installed/ @@ -184,6 +174,8 @@ check_if_source_exist() { echo "$TP_SOURCE_DIR/$1 does not exist." exit 1 fi + # Remove the build dir to clean build env + rm -rf $TP_SOURCE_DIR/$1/$BUILD_DIR echo "===== begin build $1" } @@ -517,7 +509,7 @@ build_rocksdb() { cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE - CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I ${TP_INCLUDE_DIR}/lz4" CXXFLAGS="-fPIC -Wno-deprecated-copy -Wno-stringop-truncation -Wno-pessimizing-move" LDFLAGS="-static-libstdc++ -static-libgcc" \ + CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I ${TP_INCLUDE_DIR}/lz4" CXXFLAGS="-fPIC -Wno-deprecated-copy -Wno-shadow -Wno-range-loop-construct -Wno-dangling-gsl -Wno-pessimizing-move" LDFLAGS="-static-libstdc++ -static-libgcc" \ PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib cp librocksdb.a ../../installed/lib/librocksdb.a cp -r include/rocksdb ../../installed/include/ @@ -566,7 +558,7 @@ build_flatbuffers() { cd $TP_SOURCE_DIR/$FLATBUFFERS_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR rm -rf CMakeCache.txt CMakeFiles/ - CXXFLAGS="-fPIC -Wno-class-memaccess" \ + CXXFLAGS="-fPIC " \ LDFLAGS="-static-libstdc++ -static-libgcc" \ ${CMAKE_CMD} -G "${GENERATOR}" .. ${BUILD_SYSTEM} -j $PARALLEL @@ -643,55 +635,13 @@ build_s2() { ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } -# bitshuffle build_bitshuffle() { check_if_source_exist $BITSHUFFLE_SOURCE cd $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE PREFIX=$TP_INSTALL_DIR - - # This library has significant optimizations when built with -mavx2. However, - # we still need to support non-AVX2-capable hardware. So, we build it twice, - # once with the flag and once without, and use some linker tricks to - # suffix the AVX2 symbols with '_avx2'. - arches="default avx2" - MACHINE_TYPE=$(uname -m) - # Becuase aarch64 don't support avx2, disable it. - if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then - arches="default" - fi - - to_link="" - for arch in $arches ; do - arch_flag="" - if [ "$arch" == "avx2" ]; then - arch_flag="-mavx2" - fi - tmp_obj=bitshuffle_${arch}_tmp.o - dst_obj=bitshuffle_${arch}.o - ${CC:-$DORIS_GCC_HOME/bin/gcc} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \ - "src/bitshuffle_core.c" \ - "src/bitshuffle.c" \ - "src/iochain.c" - # Merge the object files together to produce a combined .o file. - $DORIS_BIN_UTILS/ld -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o - # For the AVX2 symbols, suffix them. - if [ "$arch" == "avx2" ]; then - # Create a mapping file with ' ' on each line. - $DORIS_BIN_UTILS/nm --defined-only --extern-only $tmp_obj | while read addr type sym ; do - echo ${sym} ${sym}_${arch} - done > renames.txt - $DORIS_BIN_UTILS/objcopy --redefine-syms=renames.txt $tmp_obj $dst_obj - else - mv $tmp_obj $dst_obj - fi - to_link="$to_link $dst_obj" - done - rm -f libbitshuffle.a - $DORIS_BIN_UTILS/ar rs libbitshuffle.a $to_link mkdir -p $PREFIX/include/bitshuffle - cp libbitshuffle.a $PREFIX/lib/ - cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle.h $PREFIX/include/bitshuffle/bitshuffle.h - cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle_core.h $PREFIX/include/bitshuffle/bitshuffle_core.h + mkdir -p $BUILD_DIR && cd $BUILD_DIR + cmake ${TP_DIR}/cmake/bitshuffle-cmake -DDORIS_HOME=${DORIS_HOME} -DTP_INSTALL_DIR=${TP_INSTALL_DIR} && make install } # croaring bitmap @@ -744,7 +694,7 @@ build_orc() { cd $TP_SOURCE_DIR/$ORC_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR rm -rf CMakeCache.txt CMakeFiles/ - CXXFLAGS="-O3 -Wno-array-bounds" \ + CXXFLAGS="-O3 -Wno-array-bounds -Wno-suggest-destructor-override -Wno-suggest-override" \ ${CMAKE_CMD} -G "${GENERATOR}" ../ -DBUILD_JAVA=OFF \ -DPROTOBUF_HOME=$TP_INSTALL_DIR \ -DSNAPPY_HOME=$TP_INSTALL_DIR \ From 79a24181e3b2545ae7826f55922f8ceddf6ae6ca Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Sun, 2 Jan 2022 18:38:21 +0800 Subject: [PATCH 6/8] build thirdparty with clang --- build.sh | 15 ++----- env.sh | 25 ------------ thirdparty/build-thirdparty.sh | 73 ++++++++++++++++++++++++++++++---- 3 files changed, 70 insertions(+), 43 deletions(-) diff --git a/build.sh b/build.sh index 5c6a7f93ee859e..9043bf0b20c56e 100755 --- a/build.sh +++ b/build.sh @@ -41,12 +41,11 @@ ROOT=`cd "$ROOT"; pwd` export DORIS_HOME=${ROOT} -#export CC="/data/common/clang-11/bin/clang --gcc-toolchain=/usr/local/gcc-10.1.0" -#export CXX="/data/common/clang-11/bin/clang++ --gcc-toolchain=/usr/local/gcc-10.1.0" - -export CC="/usr/local/gcc-10.1.0/bin/gcc" -export CXX="/usr/local/gcc-10.1.0/bin/g++" +export CC="/data/common/clang-11/bin/clang-wrapper" +export CXX="/data/common/clang-11/bin/clang++-wrapper" +#export CC="/usr/local/gcc-10.1.0/bin/gcc" +#export CXX="/usr/local/gcc-10.1.0/bin/g++" . ${DORIS_HOME}/env.sh @@ -203,12 +202,6 @@ make # Clean and build Backend if [ ${BUILD_BE} -eq 1 ] ; then - # Has to set CC and CXX here - # --gcc-toolchain=/usr/local/gcc-10.1.0 - # Could set use clang here to build BE with clang - #export CC="/data/common/clang-11/bin/clang --gcc-toolchain=${DORIS_GCC_HOME}" - #export CXX="/data/common/clang-11/bin/clang++ --gcc-toolchain=${DORIS_GCC_HOME}" - CMAKE_BUILD_TYPE=${BUILD_TYPE:-Release} echo "Build Backend: ${CMAKE_BUILD_TYPE}" CMAKE_BUILD_DIR=${DORIS_HOME}/be/build_${CMAKE_BUILD_TYPE} diff --git a/env.sh b/env.sh index da74e8ac9da833..943261fef48b36 100755 --- a/env.sh +++ b/env.sh @@ -51,18 +51,6 @@ if ! ${PYTHON} --version; then fi fi -# set GCC HOME -if [[ -z ${DORIS_GCC_HOME} ]]; then - export DORIS_GCC_HOME=$(dirname $(which gcc))/.. -fi - -gcc_ver=$(${DORIS_GCC_HOME}/bin/gcc -dumpfullversion -dumpversion) -required_ver="7.3.0" -if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$required_ver" ]]; then - echo "Error: GCC version (${gcc_ver}) must be greater than or equal to ${required_ver}" - exit 1 -fi - # register keyword is forbidden to use in C++17 # the C++ code generated by flex that remove register keyword after version 2.6.0 # so we need check flex version here to avoid compilation failed @@ -73,19 +61,6 @@ if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$ exit 1 fi -<<<<<<< HEAD ->>>>>>> Could not build bitshuffle because it changed synoms -# find binutils -if test -x ${DORIS_GCC_HOME}/bin/ld; then - export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/ -else - export DORIS_BIN_UTILS=/usr/bin/ -fi - -# export CLANG COMPATIBLE FLAGS -export CLANG_COMPATIBLE_FLAGS=$(echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 | - grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' ') - # check java home if [ -z "$JAVA_HOME" ]; then export JAVACMD=$(which java) diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 0dd7b2ddd7e4f1..20a476bc104a4b 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -200,7 +200,7 @@ build_libevent() { CFLAGS="-std=c99 -fPIC -D_BSD_SOURCE -fno-omit-frame-pointer -g -ggdb -O2 -I${TP_INCLUDE_DIR}" \ CPPLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ - ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DEVENT__DISABLE_TESTS=ON \ + ${CMAKE_CMD} -G "${GENERATOR}" -DEVENT__LIBRARY_TYPE=STATIC -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DEVENT__DISABLE_TESTS=ON \ -DEVENT__DISABLE_OPENSSL=ON -DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON .. ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } @@ -346,10 +346,6 @@ build_gperftools() { CPPFLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ - LD_LIBRARY_PATH="${TP_LIB_DIR}" \ - CFLAGS="-fPIC" \ - LDFLAGS="-L${TP_LIB_DIR}" \ - LD_LIBRARY_PATH="${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR/gperftools --disable-shared --enable-static --disable-libunwind --with-pic --enable-frame-pointers make -j $PARALLEL && make install @@ -636,12 +632,74 @@ build_s2() { } build_bitshuffle() { + compiler_cmd=`which ${CC}` + compiler_tool_home=`dirname ${compiler_cmd}` + compiler_tool_home=`cd "$compiler_tool_home/.."; pwd` + + # find binutils + if test -x ${compiler_tool_home}/bin/llvm-objcopy; then + export LD_CMD=${compiler_tool_home}/bin/ld.lld + export NM_CMD=${compiler_tool_home}/bin/llvm-nm + export AR_CMD=${compiler_tool_home}/bin/llvm-ar + export OBJCOPY_CMD=${compiler_tool_home}/bin/llvm-objcopy + elif test -x ${compiler_tool_home}/bin/ld; then + export LD_CMD=${compiler_tool_home}/bin/ld + export NM_CMD=${compiler_tool_home}/bin/nm + export AR_CMD=${compiler_tool_home}/bin/ar + export OBJCOPY_CMD=${compiler_tool_home}/bin/objcopy + else + export LD_CMD=/usr/bin/ld + export NM_CMD=/usr/bin/nm + export AR_CMD=/usr/bin/ar + export OBJCOPY_CMD=/usr/bin/objcopy + fi check_if_source_exist $BITSHUFFLE_SOURCE cd $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE PREFIX=$TP_INSTALL_DIR + + # This library has significant optimizations when built with -mavx2. However, + # we still need to support non-AVX2-capable hardware. So, we build it twice, + # once with the flag and once without, and use some linker tricks to + # suffix the AVX2 symbols with '_avx2'. + arches="default avx2" + MACHINE_TYPE=$(uname -m) + # Becuase aarch64 don't support avx2, disable it. + if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then + arches="default" + fi + + to_link="" + for arch in $arches ; do + arch_flag="" + if [ "$arch" == "avx2" ]; then + arch_flag="-mavx2" + fi + tmp_obj=bitshuffle_${arch}_tmp.o + dst_obj=bitshuffle_${arch}.o + ${CC} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \ + "src/bitshuffle_core.c" \ + "src/bitshuffle.c" \ + "src/iochain.c" + # Merge the object files together to produce a combined .o file. + ${LD_CMD} -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o + # For the AVX2 symbols, suffix them. + if [ "$arch" == "avx2" ]; then + # Create a mapping file with ' ' on each line. + ${NM_CMD} --defined-only --extern-only $tmp_obj | while read addr type sym ; do + echo ${sym} ${sym}_${arch} + done > renames.txt + ${OBJCOPY_CMD} --redefine-syms=renames.txt $tmp_obj $dst_obj + else + mv $tmp_obj $dst_obj + fi + to_link="$to_link $dst_obj" + done + rm -f libbitshuffle.a + ${AR_CMD} rs libbitshuffle.a $to_link mkdir -p $PREFIX/include/bitshuffle - mkdir -p $BUILD_DIR && cd $BUILD_DIR - cmake ${TP_DIR}/cmake/bitshuffle-cmake -DDORIS_HOME=${DORIS_HOME} -DTP_INSTALL_DIR=${TP_INSTALL_DIR} && make install + cp libbitshuffle.a $PREFIX/lib/ + cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle.h $PREFIX/include/bitshuffle/bitshuffle.h + cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle_core.h $PREFIX/include/bitshuffle/bitshuffle_core.h } # croaring bitmap @@ -851,6 +909,7 @@ build_hdfs3() { check_if_source_exist $HDFS3_SOURCE cd $TP_SOURCE_DIR/$HDFS3_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR + LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \ ../bootstrap --dependency=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR make -j $PARALLEL && make install } From e9b181ac6636e2129425469ea816e2c268e18f86 Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Sun, 2 Jan 2022 22:23:16 +0800 Subject: [PATCH 7/8] add -w to disables warnings during build rocksdb --- build.sh | 8 ++++---- env.sh | 1 - thirdparty/build-thirdparty.sh | 8 +++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/build.sh b/build.sh index 9043bf0b20c56e..35e9ebc78335fc 100755 --- a/build.sh +++ b/build.sh @@ -41,11 +41,11 @@ ROOT=`cd "$ROOT"; pwd` export DORIS_HOME=${ROOT} -export CC="/data/common/clang-11/bin/clang-wrapper" -export CXX="/data/common/clang-11/bin/clang++-wrapper" +#export CC="/data/common/clang-11/bin/clang-wrapper" +#export CXX="/data/common/clang-11/bin/clang++-wrapper" -#export CC="/usr/local/gcc-10.1.0/bin/gcc" -#export CXX="/usr/local/gcc-10.1.0/bin/g++" +export CC="/usr/local/gcc-10.1.0/bin/gcc" +export CXX="/usr/local/gcc-10.1.0/bin/g++" . ${DORIS_HOME}/env.sh diff --git a/env.sh b/env.sh index 943261fef48b36..37fef4ab465c20 100755 --- a/env.sh +++ b/env.sh @@ -69,7 +69,6 @@ else export JAVA="${JAVA_HOME}/bin/java" JAVAP="${JAVA_HOME}/bin/javap" fi ->>>>>>> Could not build bitshuffle because it changed synoms # if is called from build-thirdparty.sh, no need to check these tools if test -z "${BUILD_THIRDPARTY_WIP}"; then diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 20a476bc104a4b..a80b539d52c164 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -220,7 +220,7 @@ build_openssl() { LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ LIBDIR="lib" \ - ./Configure --prefix=$TP_INSTALL_DIR --with-rand-seed=devrandom -zlib -shared ${OPENSSL_PLATFORM} + ./Configure --prefix=$TP_INSTALL_DIR --with-rand-seed=devrandom -shared ${OPENSSL_PLATFORM} make -j $PARALLEL && make install_sw # NOTE(zc): remove this dynamic library files to make libcurl static link. # If I don't remove this files, I don't known how to make libcurl link static library @@ -505,8 +505,10 @@ build_rocksdb() { cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE - CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I ${TP_INCLUDE_DIR}/lz4" CXXFLAGS="-fPIC -Wno-deprecated-copy -Wno-shadow -Wno-range-loop-construct -Wno-dangling-gsl -Wno-pessimizing-move" LDFLAGS="-static-libstdc++ -static-libgcc" \ - PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib + CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I ${TP_INCLUDE_DIR}/lz4" \ + CXXFLAGS="-fPIC -w -Wno-deprecated-copy -Wno-shadow -Wno-range-loop-construct -Wno-dangling-gsl -Wno-pessimizing-move" \ + LDFLAGS="-static-libstdc++ -static-libgcc" \ + PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib cp librocksdb.a ../../installed/lib/librocksdb.a cp -r include/rocksdb ../../installed/include/ } From f41117c17b11cb442fc7438c640f1f5f0d9a9045 Mon Sep 17 00:00:00 2001 From: guoleiyi Date: Tue, 4 Jan 2022 11:08:07 +0800 Subject: [PATCH 8/8] check doris gcc home to compatable with current build system --- build.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 35e9ebc78335fc..8c3dd81c8ee190 100755 --- a/build.sh +++ b/build.sh @@ -44,11 +44,19 @@ export DORIS_HOME=${ROOT} #export CC="/data/common/clang-11/bin/clang-wrapper" #export CXX="/data/common/clang-11/bin/clang++-wrapper" -export CC="/usr/local/gcc-10.1.0/bin/gcc" -export CXX="/usr/local/gcc-10.1.0/bin/g++" +#export CC="/usr/local/gcc-10.1.0/bin/gcc" +#export CXX="/usr/local/gcc-10.1.0/bin/g++" . ${DORIS_HOME}/env.sh +if [[ -z ${CC} ]]; then + if [[ -z ${DORIS_GCC_HOME} ]]; then + DORIS_GCC_HOME=$(dirname $(which gcc))/.. + fi + export CC="${DORIS_GCC_HOME}/bin/gcc" + export CXX="${DORIS_GCC_HOME}/bin/g++" +fi + # Check args usage() { echo "