From 3bd141496219b40cd1a3c409698fd9db37d3ccb8 Mon Sep 17 00:00:00 2001 From: lenmom Date: Mon, 20 May 2019 11:38:25 +0800 Subject: [PATCH 1/3] fix arrow break build error on Ubuntu 18.04 OS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [ 38%] Building CXX object src/arrow/CMakeFiles/arrow_objlib.dir/scalar.cc.o /media/lenmom/新加卷/workspace/open-source/incubator-doris-0.10.0-rc01/thirdparty/src/arrow-apache-arrow-0.13.0/cpp/src/arrow/util/compression_snappy.cc:24:10: fatal error: snappy.h: No such file or directory #include ^~~~~~~~~~ compilation terminated. src/arrow/CMakeFiles/arrow_objlib.dir/build.make:110: recipe for target 'src/arrow/CMakeFiles/arrow_objlib.dir/util/compression_snappy.cc.o' failed make[2]: *** [src/arrow/CMakeFiles/arrow_objlib.dir/util/compression_snappy.cc.o] Error 1 make[2]: *** Waiting for unfinished jobs.... [ 38%] Building CXX object src/arrow/CMakeFiles/arrow_objlib.dir/tensor.cc.o CMakeFiles/Makefile2:860: recipe for target 'src/arrow/CMakeFiles/arrow_objlib.dir/all' failed make[1]: *** [src/arrow/CMakeFiles/arrow_objlib.dir/all] Error 2 Makefile:140: recipe for target 'all' failed make: *** [all] Error 2 cp: cannot stat './zstd_ep-install/lib64/libzstd.a': No such file or directory --- thirdparty/build-thirdparty.sh | 40 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index a9090aba53d255..a3f45bbd54d530 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -129,7 +129,7 @@ check_if_source_exist() { echo "dir should specified to check if exist." exit 1 fi - + if [ ! -d $TP_SOURCE_DIR/$1 ];then echo "$TP_SOURCE_DIR/$1 does not exist." exit 1 @@ -142,7 +142,7 @@ check_if_archieve_exist() { echo "archieve should specified to check if exist." exit 1 fi - + if [ ! -f $TP_SOURCE_DIR/$1 ];then echo "$TP_SOURCE_DIR/$1 does not exist." exit 1 @@ -154,7 +154,7 @@ build_libevent() { check_if_source_exist $LIBEVENT_SOURCE cd $TP_SOURCE_DIR/$LIBEVENT_SOURCE if [ ! -f configure ]; then - ./autogen.sh + ./autogen.sh fi CFLAGS="-std=c99 -fPIC -D_BSD_SOURCE -fno-omit-frame-pointer -g -ggdb -O2 -I${TP_INCLUDE_DIR}" \ @@ -328,12 +328,14 @@ build_snappy() { make -j$PARALLEL && make install if [ -f $TP_INSTALL_DIR/lib64/libsnappy.a ]; then mkdir -p $TP_INSTALL_DIR/lib && cp $TP_INSTALL_DIR/lib64/libsnappy.a $TP_INSTALL_DIR/lib/libsnappy.a - #build for libarrow.a - cp $TP_INCLUDE_DIR/snappy/snappy-c.h $TP_INCLUDE_DIR/snappy-c.h && \ - cp $TP_INCLUDE_DIR/snappy/snappy-sinksource.h $TP_INCLUDE_DIR/snappy-sinksource.h && \ - cp $TP_INCLUDE_DIR/snappy/snappy-stubs-public.h $TP_INCLUDE_DIR/snappy-stubs-public.h && \ - cp $TP_INCLUDE_DIR/snappy/snappy.h $TP_INCLUDE_DIR/snappy.h + fi + + #build for libarrow.a + cp $TP_INCLUDE_DIR/snappy/snappy-c.h $TP_INCLUDE_DIR/snappy-c.h && \ + cp $TP_INCLUDE_DIR/snappy/snappy-sinksource.h $TP_INCLUDE_DIR/snappy-sinksource.h && \ + cp $TP_INCLUDE_DIR/snappy/snappy-stubs-public.h $TP_INCLUDE_DIR/snappy-stubs-public.h && \ + cp $TP_INCLUDE_DIR/snappy/snappy.h $TP_INCLUDE_DIR/snappy.h } # gperftools @@ -341,7 +343,7 @@ build_gperftools() { check_if_source_exist $GPERFTOOLS_SOURCE cd $TP_SOURCE_DIR/$GPERFTOOLS_SOURCE if [ ! -f configure ]; then - ./autogen.sh + ./autogen.sh fi CPPFLAGS="-I${TP_INCLUDE_DIR}" \ @@ -389,7 +391,7 @@ build_bzip() { build_lzo2() { check_if_source_exist $LZO2_SOURCE cd $TP_SOURCE_DIR/$LZO2_SOURCE - + CPPFLAGS="-I${TP_INCLUDE_DIR} -fPIC" \ LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ @@ -401,7 +403,7 @@ build_lzo2() { build_curl() { check_if_source_exist $CURL_SOURCE cd $TP_SOURCE_DIR/$CURL_SOURCE - + CPPFLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" LIBS="-lcrypto -lssl -lcrypto -ldl" \ CFLAGS="-fPIC" \ @@ -414,7 +416,7 @@ build_curl() { build_re2() { check_if_source_exist $RE2_SOURCE cd $TP_SOURCE_DIR/$RE2_SOURCE - + $CMAKE_CMD -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR make -j$PARALLEL install } @@ -425,7 +427,7 @@ build_boost() { cd $TP_SOURCE_DIR/$BOOST_SOURCE echo "using gcc : doris : ${CXX} ; " > tools/build/src/user-config.jam - ./bootstrap.sh --prefix=$TP_INSTALL_DIR + ./bootstrap.sh --prefix=$TP_INSTALL_DIR ./b2 --toolset=gcc-doris link=static -d0 -j$PARALLEL --without-mpi --without-graph --without-graph_parallel --without-python cxxflags="-std=c++11 -fPIC -I$TP_INCLUDE_DIR -L$TP_LIB_DIR" install } @@ -459,7 +461,7 @@ build_mysql() { cp -R ../include/* ../../../installed/include/mysql/ cp ../libbinlogevents/export/binary_log_types.h ../../../installed/include/mysql/ echo "mysql headers are installed." - + # copy libmysqlclient.a cp libmysql/libmysqlclient.a ../../../installed/lib/ echo "mysql client lib is installed." @@ -531,7 +533,7 @@ build_arrow() { export ARROW_LZ4_URL=${TP_SOURCE_DIR}/${LZ4_NAME} export ARROW_URIPARSER_URL=${TP_SOURCE_DIR}/${URIPARSER_NAME} export ARROW_ZSTD_URL=${TP_SOURCE_DIR}/${ZSTD_NAME} - + cmake -DARROW_PARQUET=ON -DARROW_IPC=OFF -DARROW_BUILD_SHARED=OFF \ -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ -DARROW_BOOST_USE_SHARED=OFF -DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=$TP_INSTALL_DIR \ @@ -547,12 +549,16 @@ build_arrow() { cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc.a $TP_INSTALL_DIR/lib64/libbrotlienc.a cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec.a $TP_INSTALL_DIR/lib64/libbrotlidec.a cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon.a $TP_INSTALL_DIR/lib64/libbrotlicommon.a - cp -rf ./zstd_ep-install/lib64/libzstd.a $TP_INSTALL_DIR/lib64/libzstd.a + if [ -f ./zstd_ep-install/lib64/libzstd.a ]; then + cp -rf ./zstd_ep-install/lib64/libzstd.a $TP_INSTALL_DIR/lib64/libzstd.a + else + cp -rf ./zstd_ep-install/lib/libzstd.a $TP_INSTALL_DIR/lib64/libzstd.a + fi cp -rf ./double-conversion_ep/src/double-conversion_ep/lib/libdouble-conversion.a $TP_INSTALL_DIR/lib64/libdouble-conversion.a cp -rf ./uriparser_ep-install/lib/liburiparser.a $TP_INSTALL_DIR/lib64/liburiparser.a } -build_llvm +build_llvm build_libevent build_zlib build_lz4 From 6e3c196e10904c92a935555564adf1442427af55 Mon Sep 17 00:00:00 2001 From: lenmom Date: Wed, 7 Aug 2019 19:53:39 +0800 Subject: [PATCH 2/3] Update build-thirdparty.sh --- thirdparty/build-thirdparty.sh | 86 ++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 5 deletions(-) diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index a3f45bbd54d530..92513ae224e01e 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -173,7 +173,7 @@ build_openssl() { CFLAGS="-fPIC" \ LIBDIR="lib" \ ./Configure --prefix=$TP_INSTALL_DIR -zlib -shared linux-x86_64 - make -j$PARALLEL && make install + make && make install if [ -f $TP_INSTALL_DIR/lib64/libcrypto.a ]; then mkdir -p $TP_INSTALL_DIR/lib && \ cp $TP_INSTALL_DIR/lib64/libcrypto.a $TP_INSTALL_DIR/lib/libcrypto.a && \ @@ -321,7 +321,7 @@ build_snappy() { mkdir build -p && cd build rm -rf CMakeCache.txt CMakeFiles/ - $CMAKE_CMD -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ + CFLAGS="-O3" CXXFLAGS="-O3" $CMAKE_CMD -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ -DCMAKE_POSITION_INDEPENDENT_CODE=On \ -DCMAKE_INSTALL_INCLUDEDIR=$TP_INCLUDE_DIR/snappy \ -DSNAPPY_BUILD_TESTS=0 ../ @@ -489,7 +489,7 @@ build_brpc() { rm -rf CMakeCache.txt CMakeFiles/ LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \ $CMAKE_CMD -v -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ - -DBRPC_WITH_GLOG=ON -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \ + -DBRPC_WITH_GLOG=ON -DWITH_GLOG=ON -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \ -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib;$TP_INSTALL_DIR/lib64" \ -DPROTOBUF_PROTOC_EXECUTABLE=$TP_INSTALL_DIR/bin/protoc \ -DProtobuf_PROTOC_EXECUTABLE=$TP_INSTALL_DIR/bin/protoc .. @@ -520,7 +520,7 @@ build_librdkafka() { CPPFLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ - ./configure --prefix=$TP_INSTALL_DIR --enable-static --disable-ssl --disable-sasl + ./configure --prefix=$TP_INSTALL_DIR --enable-static --disable-sasl make -j$PARALLEL && make install } @@ -558,6 +558,80 @@ build_arrow() { cp -rf ./uriparser_ep-install/lib/liburiparser.a $TP_INSTALL_DIR/lib64/liburiparser.a } +# s2 +build_s2() { + check_if_source_exist $S2_SOURCE + cd $TP_SOURCE_DIR/s2geometry-0.9.0 + mkdir build -p && cd build + rm -rf CMakeCache.txt CMakeFiles/ + CXXFLAGS="-O3" \ + LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \ + $CMAKE_CMD -v -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ + -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \ + -DBUILD_SHARED_LIBS=OFF \ + -DGFLAGS_ROOT_DIR="$TP_INSTALL_DIR/include" \ + -DWITH_GFLAGS=ON \ + -DGLOG_ROOT_DIR="$TP_INSTALL_DIR/include" \ + -DWITH_GLOG=ON \ + -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib;$TP_INSTALL_DIR/lib64" .. + make -j$PARALLEL && make install +} + +# bitshuffle +build_bitshuffle() { + check_if_source_exist $BITSHUFFLE_SOURCE + cd $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE + PREFIX=$TP_INSTALL_DIR + arches="default avx2" + to_link="" + for arch in $arches ; do + arch_flag="" + if [ "$arch" == "avx2" ]; then + arch_flag="-mavx2" + fi + tmp_obj=bitshuffle_${arch}_tmp.o + dst_obj=bitshuffle_${arch}.o + ${CC:-gcc} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \ + "src/bitshuffle_core.c" \ + "src/bitshuffle.c" \ + "src/iochain.c" + # Merge the object files together to produce a combined .o file. + ld -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o + # For the AVX2 symbols, suffix them. + if [ "$arch" == "avx2" ]; then + # Create a mapping file with ' ' on each line. + nm --defined-only --extern-only $tmp_obj | while read addr type sym ; do + echo ${sym} ${sym}_${arch} + done > renames.txt + objcopy --redefine-syms=renames.txt $tmp_obj $dst_obj + else + mv $tmp_obj $dst_obj + fi + to_link="$to_link $dst_obj" + done + rm -f libbitshuffle.a + ar rs libbitshuffle.a $to_link + mkdir -p $PREFIX/include/bitshuffle + cp libbitshuffle.a $PREFIX/lib/ + cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle.h $PREFIX/include/bitshuffle/bitshuffle.h + cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle_core.h $PREFIX/include/bitshuffle/bitshuffle_core.h +} + +# croaring bitmap +build_croaringbitmap() { + check_if_source_exist $CROARINGBITMAP_SOURCE + cd $TP_SOURCE_DIR/CRoaring-0.2.60 + mkdir build -p && cd build + rm -rf CMakeCache.txt CMakeFiles/ + CXXFLAGS="-O3" \ + LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \ + $CMAKE_CMD -v -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ + -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \ + -DENABLE_ROARING_TESTS=OFF \ + -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib;$TP_INSTALL_DIR/lib64" .. + make -j$PARALLEL && make install +} + build_llvm build_libevent build_zlib @@ -582,6 +656,8 @@ build_brpc build_rocksdb build_librdkafka build_arrow +build_s2 +build_bitshuffle +build_croaringbitmap echo "Finihsed to build all thirdparties" - From eeb5d223c33be1fa84876c9588d5862dc6a8271f Mon Sep 17 00:00:00 2001 From: lenmom Date: Wed, 7 Aug 2019 19:58:55 +0800 Subject: [PATCH 3/3] fix break build error for preadv by include header file for this. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/env/env_posix.cpp:99:27: error: ‘preadv’ was not declared in this scope RETRY_ON_EINTR(r, preadv(fd, iov + completed_iov, iov_count, cur_offset)); ^ /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/gutil/macros.h:269:12: note: in definition of macro ‘RETRY_ON_EINTR’ (err) = (expr); \ ^~~~ /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/env/env_posix.cpp:99:27: note: suggested alternative: ‘pread’ RETRY_ON_EINTR(r, preadv(fd, iov + completed_iov, iov_count, cur_offset)); ^ /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/gutil/macros.h:269:12: note: in definition of macro ‘RETRY_ON_EINTR’ (err) = (expr); \ ^~~~ /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/env/env_posix.cpp: In function ‘doris::Status doris::do_writev_at(int, const string&, uint64_t, const doris::Slice*, size_t, size_t*)’: /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/env/env_posix.cpp:156:27: error: ‘pwritev’ was not declared in this scope RETRY_ON_EINTR(w, pwritev(fd, iov + completed_iov, iov_count, cur_offset)); ^ /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/gutil/macros.h:269:12: note: in definition of macro ‘RETRY_ON_EINTR’ (err) = (expr); \ ^~~~ /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/env/env_posix.cpp:156:27: note: suggested alternative: ‘pwrite’ RETRY_ON_EINTR(w, pwritev(fd, iov + completed_iov, iov_count, cur_offset)); ^ /media/lenmom/新加卷/workspace/open-source/incubator-doris-DORIS-0.11.5-alpha/be/src/gutil/macros.h:269:12: note: in definition of macro ‘RETRY_ON_EINTR’ (err) = (expr); \ ^~~~ src/env/CMakeFiles/Env.dir/build.make:62: recipe for target 'src/env/CMakeFiles/Env.dir/env_posix.cpp.o' failed make[2]: *** [src/env/CMakeFiles/Env.dir/env_posix.cpp.o] Error 1 CMakeFiles/Makefile2:361: recipe for target 'src/env/CMakeFiles/Env.dir/all' failed make[1]: *** [src/env/CMakeFiles/Env.dir/all] Error 2 make[1]: *** Waiting for unfinished jobs.... [ 4%] Building CXX object src/exprs/CMakeFiles/Exprs.dir/conditional_functions.cpp.o --- be/src/env/env_posix.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/env/env_posix.cpp b/be/src/env/env_posix.cpp index 3bc6c08fafbb2b..f86fc5457ce178 100644 --- a/be/src/env/env_posix.cpp +++ b/be/src/env/env_posix.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include "common/logging.h" #include "gutil/macros.h"