diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index ce208d8abc6e4b..66616e01d71781 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -294,17 +294,20 @@ set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib add_library(minzip STATIC IMPORTED) set_target_properties(minzip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libminizip.a) -add_library(hdfs3 STATIC IMPORTED) -set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a) +if (ARCH_AMD64) + # libhdfs3 only support x86 or amd64 + add_library(hdfs3 STATIC IMPORTED) + set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a) -add_library(gsasl STATIC IMPORTED) -set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a) + add_library(gsasl STATIC IMPORTED) + set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a) -add_library(xml2 STATIC IMPORTED) -set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a) + add_library(xml2 STATIC IMPORTED) + set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a) -add_library(lzma STATIC IMPORTED) -set_target_properties(lzma PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/liblzma.a) + add_library(lzma STATIC IMPORTED) + set_target_properties(lzma PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/liblzma.a) +endif() find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin) @@ -434,7 +437,7 @@ set(WL_END_GROUP "-Wl,--end-group") set(AWS_LIBS aws-sdk-s3 aws-sdk-core aws-checksums aws-c-io aws-c-event-stream aws-c-common aws-c-cal aws-s2n) -# Set Palo libraries +# Set Doris libraries set(DORIS_LINK_LIBS ${WL_START_GROUP} Agent @@ -459,10 +462,10 @@ set(DORIS_LINK_LIBS ${WL_END_GROUP} ) -# Set thirdparty libraries -set(DORIS_DEPENDENCIES - ${DORIS_DEPENDENCIES} - ${WL_START_GROUP} +# COMMON_THIRDPARTY are thirdparty dependencies that can run on all platform +# When adding new dependencies, If you don’t know if it can run on all platforms, +# add it here first. +set(COMMON_THIRDPARTY rocksdb librdkafka_cpp librdkafka @@ -506,14 +509,35 @@ set(DORIS_DEPENDENCIES odbc cctz minzip + ${AWS_LIBS} +) + +# thirdparties dependescies that can only run on X86 platform +set(X86_DEPENDENCIES + ${COMMON_THIRDPARTY} hdfs3 gsasl xml2 lzma - ${AWS_LIBS} - ${WL_END_GROUP} ) +if(ARCH_AARCH64) + # Set thirdparty libraries + set(DORIS_DEPENDENCIES + ${DORIS_DEPENDENCIES} + ${WL_START_GROUP} + ${COMMON_THIRDPARTY} + ${WL_END_GROUP} + ) +else() + set(DORIS_DEPENDENCIES + ${DORIS_DEPENDENCIES} + ${WL_START_GROUP} + ${X86_DEPENDENCIES} + ${WL_END_GROUP} + ) +endif() + if(WITH_LZO) set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} lzo @@ -526,6 +550,8 @@ if (WITH_MYSQL) ) endif() +message(STATUS "DORIS_DEPENDENCIES is ${DORIS_DEPENDENCIES}") + # Add all external dependencies. They should come after the palo libs. # static link gcc's lib set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index c958c67c9f5cc5..27ef5ed911e10c 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -40,7 +40,6 @@ set(EXEC_FILES hash_join_node.cpp hash_join_node_ir.cpp hash_table.cpp - hdfs_file_reader.cpp local_file_reader.cpp merge_node.cpp merge_join_node.cpp @@ -107,6 +106,13 @@ set(EXEC_FILES s3_writer.cpp ) +if (ARCH_AMD64) + set(EXEC_FILES + ${EXEC_FILES} + hdfs_file_reader.cpp + ) +endif() + if (WITH_MYSQL) set(EXEC_FILES ${EXEC_FILES} diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp index 5d57a703c7afff..6287705bed8641 100644 --- a/be/src/exec/broker_scanner.cpp +++ b/be/src/exec/broker_scanner.cpp @@ -24,7 +24,6 @@ #include "exec/buffered_reader.h" #include "exec/decompressor.h" #include "exec/exec_node.h" -#include "exec/hdfs_file_reader.h" #include "exec/local_file_reader.h" #include "exec/plain_text_line_reader.h" #include "exec/s3_reader.h" @@ -40,6 +39,10 @@ #include "runtime/tuple.h" #include "util/utf8_check.h" +#if defined(__x86_64__) + #include "exec/hdfs_file_reader.h" +#endif + namespace doris { BrokerScanner::BrokerScanner(RuntimeState* state, RuntimeProfile* profile, @@ -163,12 +166,16 @@ Status BrokerScanner::open_file_reader() { break; } case TFileType::FILE_HDFS: { +#if defined(__x86_64__) BufferedReader* file_reader = new BufferedReader(new HdfsFileReader(range.hdfs_params, range.path, start_offset), config::remote_storage_read_buffer_mb * 1024 * 1024); RETURN_IF_ERROR(file_reader->open()); _cur_file_reader = file_reader; break; +#else + return Status::InternalError("HdfsFileReader do not support on non x86 platform"); +#endif } case TFileType::FILE_BROKER: { BrokerReader* broker_reader = diff --git a/be/src/exec/parquet_scanner.cpp b/be/src/exec/parquet_scanner.cpp index 8d0bf57be7ca16..72e460d5141221 100644 --- a/be/src/exec/parquet_scanner.cpp +++ b/be/src/exec/parquet_scanner.cpp @@ -36,13 +36,16 @@ #include "exprs/expr.h" #include "exec/text_converter.h" #include "exec/text_converter.hpp" -#include "exec/hdfs_file_reader.h" #include "exec/local_file_reader.h" #include "exec/broker_reader.h" #include "exec/buffered_reader.h" #include "exec/decompressor.h" #include "exec/parquet_reader.h" +#if defined(__x86_64__) + #include "exec/hdfs_file_reader.h" +#endif + namespace doris { ParquetScanner::ParquetScanner(RuntimeState* state, RuntimeProfile* profile, @@ -128,9 +131,13 @@ Status ParquetScanner::open_next_reader() { break; } case TFileType::FILE_HDFS: { +#if defined(__x86_64__) file_reader.reset(new HdfsFileReader( range.hdfs_params, range.path, range.start_offset)); break; +#else + return Status::InternalError("HdfsFileReader do not support on non x86 platform"); +#endif } case TFileType::FILE_BROKER: { int64_t file_size = 0; diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js index 75cfe7148a9bbd..b1d7660f3bf8b4 100644 --- a/docs/.vuepress/sidebar/en.js +++ b/docs/.vuepress/sidebar/en.js @@ -28,6 +28,7 @@ module.exports = [ directoryPath: "installing/", children: [ "compilation", + "compilation-arm", "install-deploy", "upgrade", ], diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js index cfbe0a4e7db8ce..c8ff8582915cd9 100644 --- a/docs/.vuepress/sidebar/zh-CN.js +++ b/docs/.vuepress/sidebar/zh-CN.js @@ -28,6 +28,7 @@ module.exports = [ directoryPath: "installing/", children: [ "compilation", + "compilation-arm", "install-deploy", "upgrade", ], diff --git a/docs/en/installing/compilation-arm.md b/docs/en/installing/compilation-arm.md new file mode 100644 index 00000000000000..58a5d40c4f7b49 --- /dev/null +++ b/docs/en/installing/compilation-arm.md @@ -0,0 +1,223 @@ +--- +{ + "title": "Compile on ARM platform", + "language": "en" +} +--- + + + + +# Compile and Run Doris on ARM64 + KylinOS. + +This document describes how to compile Doris on the ARM64 platform. + +Note that this document is only a guide document. Other errors may occur when compiling in different environments. + +## Software and hardware environment + +1. KylinOS version: + + ``` + $> cat /etc/.kyinfo + name=Kylin-Server + milestone=10-SP1-Release-Build04-20200711 + arch=arm64 + beta=False + time=2020-07-11 17:16:54 + dist_id=Kylin-Server-10-SP1-Release-Build04-20200711-arm64-2020-07-11 17:16:54 + ``` + +2. CPU model + + ``` + $> cat /proc/cpuinfo + model name: Phytium,FT-2000+/64 + ``` + +3. Doris version + + commit 68bab73 + +## Compilation tool installation (no network) + +In the example, all tools are installed in the `/home/doris/tools/installed/` directory. + +Please obtain the required installation package first under network conditions. + +### 1. Install gcc10 + +Download gcc-10.1.0 + +``` +wget https://mirrors.tuna.tsinghua.edu.cn/gnu/gcc/gcc-10.1.0/gcc-10.1.0.tar.gz +``` + +After unzipping, check the dependencies in `contrib/download_prerequisites` and download: + +``` +http://gcc.gnu.org/pub/gcc/infrastructure/gmp-6.1.0.tar.bz2 +http://gcc.gnu.org/pub/gcc/infrastructure/mpfr-3.1.4.tar.bz2 +http://gcc.gnu.org/pub/gcc/infrastructure/mpc-1.0.3.tar.gz +http://gcc.gnu.org/pub/gcc/infrastructure/isl-0.18.tar.bz2 +``` + +Unzip these four dependencies, then move to the gcc-10.1.0 source directory and rename them to gmp, isl, mpc, mpfr. + +Download and install automake-1.15 (because gcc10 will find automake 1.15 version during compilation) + +``` +https://ftp.gnu.org/gnu/automake/automake-1.15.tar.gz +tar xzf automake-1.15.tar.gz +./configure --prefix=/home/doris/tools/installed +make && make install +export PATH=/home/doris/tools/installed/bin:$PATH +``` + +Compile GCC10: + +``` +cd gcc-10.1.0 +./configure --prefix=/home/doris/tools/installed +make -j && make install +``` + +Compile time is longer. + +### 2. Install other compilation components + +1. jdk-8u291-linux-aarch64.tar.gz + + `https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html` + + No need to compile, just use it out of the box. + +2. cmake-3.19.8-Linux-aarch64.tar.gz + + `https://cmake.org/download/` + + No need to compile, just use it out of the box + +3. apache-maven-3.8.1-bin.tar.gz + + `https://maven.apache.org/download.cgi` + + No need to compile, just use it out of the box + +4. nodejs 16.3.0 + + `https://nodejs.org/dist/v16.3.0/node-v16.3.0-linux-arm64.tar.xz` + + No need to compile, just use it out of the box + +5. libtool-2.4.6.tar.gz + + For compiling third-party components, although the system may come with libtool, libtool needs to be together with automake, so it is not easy to cause problems. + + ``` + https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz + cd libtool-2.4.6/ + ./configure --prefix=/home/doris/tools/installed + make -j && make install + ``` + +6. binutils-2.36.tar.xz (obtain bdf.h) + + ``` + https://ftp.gnu.org/gnu/binutils/binutils-2.36.tar.bz2 + ./configure --prefix=/home/doris/tools/installed + make -j && make install + ``` + +7. Libiberty (for compiling BE) + + The source code of this library is under the source code package of gcc-10.1.0 + ``` + cd gcc-10.1.0/libiberty/ + ./configure --prefix=/home/doris/tools/installed + make + ``` + + After compilation, libiberty.a will be generated, which can be moved to the lib64 directory of Doris' thirdparty. + +### 3. Compile third-party libraries + +Suppose Doris source code is under `/home/doris/doris-src/`. + +1. Manually download all third-party libraries and place them in the thirdparty/src directory. +2. Add `custom_env.sh` in the Doris source directory and add the following content + + ``` + export DORIS_THIRDPARTY=/home/doris/doris-src/thirdparty/ + export JAVA_HOME=/home/doris/tools/jdk1.8.0_291/ + export DORIS_GCC_HOME=/home/doris/tools/installed/ + export PATCH_COMPILER_RT=true + ``` + + Pay attention to replace the corresponding directory + +3. Modify part of the content in build-thirdparty.sh + + 1. Close `build_mysql` and `build_libhdfs3` + + mysql is no longer needed. However, libhdfs3 does not support arm architecture for the time being, so running Doris in arm does not support direct access to hdfs through libhdfs3, and requires a broker. + + 2. Add the configure parameter in `build_curl`: `--without-libpsl`. If it is not added, an error may be reported during the linking phase of the final compilation of Doris BE: `undefined reference to ‘psl_is_cookie_domain_acceptable'` + +4. Execute build-thirdparty.sh. Here are only possible errors + + * `error: narrowing conversion of'-1' from'int' to'char' [-Wnarrowing]` + + There will be an error when compiling brpc 0.9.7. The solution is to add `-Wno-narrowing` in `CMAKE_CXX_FLAGS` of CMakeLists.txt of brpc. This problem has been fixed in the brpc master code: + + `https://github.com/apache/incubator-brpc/issues/1091` + + * `libz.a(deflate.o): relocation R_AARCH64_ADR_PREL_PG_HI21 against symbol `z_errmsg' which may bind externally can not be used when making a shared object; recompile with -fPIC` + + There will be errors when compiling brpc 0.9.7, and libcrypto will also report similar errors. The reason is unknown. It seems that under aarch64, brpc needs to link the dynamic zlib and crypto libraries. But when we compile these two third-party libraries, we only compiled .a static files. Solution: Recompile zlib and openssl to generate .so dynamic library: + + Open `build-thirdparty.sh`, find the `build_zlib` function, and change: + + ``` + ./configure --prefix=$TP_INSTALL_DIR --static + Just change to + ./configure --prefix=$TP_INSTALL_DIR + ``` + + Find `build_openssl` and comment out the following parts: + + ``` + #if [-f $TP_INSTALL_DIR/lib64/libcrypto.so ]; then + # rm -rf $TP_INSTALL_DIR/lib64/libcrypto.so* + #fi + #if [-f $TP_INSTALL_DIR/lib64/libssl.so ]; then + # rm -rf $TP_INSTALL_DIR/lib64/libssl.so* + #fi + ``` + + Then go to `build-thirdparty.sh`, comment out other `build_xxx`, open only `build_zlib` and `build_openssl`, and `build_brpc` and later `build_xxx`. Then re-execute `build-thirdparty.sh`. + + * The compilation is stuck at a certain stage. + + Not sure why. Solution: Rerun `build-thirdparty.sh`. `build-thirdparty.sh` can be executed repeatedly. + +### 4. Compile Doris source code + +Just execute sh build.sh. diff --git a/docs/zh-CN/installing/compilation-arm.md b/docs/zh-CN/installing/compilation-arm.md new file mode 100644 index 00000000000000..7a26f17c7321de --- /dev/null +++ b/docs/zh-CN/installing/compilation-arm.md @@ -0,0 +1,223 @@ +--- +{ + "title": "在ARM平台上编译", + "language": "zh-CN" +} +--- + + + + +# ARM64 + KylinOS 编译运行 Doris + +本文档介绍如何在 ARM64 平台上编译 Doris。 + +注意，该文档仅作为指导性文档。在不同环境中编译可能出现其他错误。 + +## 软硬件环境 + +1. KylinOS 版本： + + ``` + $> cat /etc/.kyinfo + name=Kylin-Server + milestone=10-SP1-Release-Build04-20200711 + arch=arm64 + beta=False + time=2020-07-11 17:16:54 + dist_id=Kylin-Server-10-SP1-Release-Build04-20200711-arm64-2020-07-11 17:16:54 + ``` + +2. CPU型号 + + ``` + $> cat /proc/cpuinfo + model name : Phytium,FT-2000+/64 + ``` + +3. Doris 版本 + + commit 68bab73 + +## 编译工具安装（无网络） + +示例中，所有工具安装在在 `/home/doris/tools/installed/` 目录下。 + +所需安装包请先在有网络情况下获取。 + +### 1. 安装gcc10 + +下载 gcc-10.1.0 + +``` +wget https://mirrors.tuna.tsinghua.edu.cn/gnu/gcc/gcc-10.1.0/gcc-10.1.0.tar.gz +``` + +解压后，在 `contrib/download_prerequisites` 查看依赖并下载： + +``` +http://gcc.gnu.org/pub/gcc/infrastructure/gmp-6.1.0.tar.bz2 +http://gcc.gnu.org/pub/gcc/infrastructure/mpfr-3.1.4.tar.bz2 +http://gcc.gnu.org/pub/gcc/infrastructure/mpc-1.0.3.tar.gz +http://gcc.gnu.org/pub/gcc/infrastructure/isl-0.18.tar.bz2 +``` + +解压这四个依赖，然后移动到 gcc-10.1.0 源码目录下，并重命名为 gmp、isl、mpc、mpfr。 + +下载并安装 automake-1.15（因为gcc10编译过程中会查找automake 1.15 版本） + +``` +https://ftp.gnu.org/gnu/automake/automake-1.15.tar.gz +tar xzf automake-1.15.tar.gz +./configure --prefix=/home/doris/tools/installed +make && make install +export PATH=/home/doris/tools/installed/bin:$PATH +``` + +编译GCC10: + +``` +cd gcc-10.1.0 +./configure --prefix=/home/doris/tools/installed +make -j && make install +``` + +编译时间较长。 + +### 2. 安装其他编译组件 + +1. jdk-8u291-linux-aarch64.tar.gz + + `https://www.oracle.com/java/technologies/javase/javase-jdk8-downloads.html` + + 无需编译，开箱即用。 + +2. cmake-3.19.8-Linux-aarch64.tar.gz + + `https://cmake.org/download/` + + 无需编译，开箱即用 + +3. apache-maven-3.8.1-bin.tar.gz + + `https://maven.apache.org/download.cgi` + + 无需编译，开箱即用 + +4. nodejs 16.3.0 + + `https://nodejs.org/dist/v16.3.0/node-v16.3.0-linux-arm64.tar.xz` + + 无需编译，开箱即用 + +5. libtool-2.4.6.tar.gz + + 编译第三方组件用，虽然系统可能自带了libtool，但是libtool需要和automake在一起，这样不容易出问题。 + + ``` + https://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz + cd libtool-2.4.6/ + ./configure --prefix=/home/doris/tools/installed + make -j && make install + ``` + +6. binutils-2.36.tar.xz（获取bdf.h） + + ``` + https://ftp.gnu.org/gnu/binutils/binutils-2.36.tar.bz2 + ./configure --prefix=/home/doris/tools/installed + make -j && make install + ``` + +7. libiberty（编译BE用） + + 这个库的源码就在 gcc-10.1.0 的源码包下 + ``` + cd gcc-10.1.0/libiberty/ + ./configure --prefix=/home/doris/tools/installed + make + ``` + + 编译后会产生 libiberty.a，后续移动到 Doris 的thirdparty 的 lib64 目录中即可。 + +### 3. 编译第三方库 + +假设Doris源码在 `/home/doris/doris-src/` 下。 + +1. 手动下载所有第三方库并放在 thirdparty/src 目录下。 +2. 在Doris源码目录下新增 `custom_env.sh` 并添加如下内容 + + ``` + export DORIS_THIRDPARTY=/home/doris/doris-src/thirdparty/ + export JAVA_HOME=/home/doris/tools/jdk1.8.0_291/ + export DORIS_GCC_HOME=/home/doris/tools/installed/ + export PATCH_COMPILER_RT=true + ``` + + 注意替换对应的目录 + +3. 修改 build-thirdparty.sh 中的部分内容 + + 1. 关闭 `build_mysql` 和 `build_libhdfs3` + + mysql 不在需要。而 libhdfs3 暂不支持 arm 架构，所以在arm中运行Doris，暂不支持通过 libhdfs3 直接访问 hdfs，需要通过broker。 + + 2. 在 `build_curl` 中增加 configure 参数：`--without-libpsl`。如果不添加，则在最终编译Doris BE的链接阶段，可能报错：`undefined reference to ‘psl_is_cookie_domain_acceptable'` + +4. 执行 build-thirdparty.sh。这里仅列举可能出现的错误 + + * ` error: narrowing conversion of '-1' from 'int' to 'char' [-Wnarrowing]` + + 编译brpc 0.9.7 时会出现错误，解决方案，在 brpc 的 CMakeLists.txt 的 `CMAKE_CXX_FLAGS` 中添加 `-Wno-narrowing`。brpc master 代码中已经修复这个问题： + + `https://github.com/apache/incubator-brpc/issues/1091` + + * `libz.a(deflate.o): relocation R_AARCH64_ADR_PREL_PG_HI21 against symbol `z_errmsg' which may bind externally can not be used when making a shared object; recompile with -fPIC` + + 编译brpc 0.9.7 时会出现错误，还有 libcrypto 也会报类似错误。原因未知，似乎在 aarch64 下，brpc 需要链接动态的 zlib 和 crypto 库。但是我们在编译这两个第三方库时，都只编译的了 .a 静态文件。解决方案：重新编译zlib和 openssl 生成.so 动态库： + + 打开 `build-thirdparty.sh`，找到 `build_zlib` 函数，将： + + ``` + ./configure --prefix=$TP_INSTALL_DIR --static + 就改为 + ./configure --prefix=$TP_INSTALL_DIR + ``` + + 找到 `build_openssl`，将以下部分注释掉： + + ``` + #if [ -f $TP_INSTALL_DIR/lib64/libcrypto.so ]; then + # rm -rf $TP_INSTALL_DIR/lib64/libcrypto.so* + #fi + #if [ -f $TP_INSTALL_DIR/lib64/libssl.so ]; then + # rm -rf $TP_INSTALL_DIR/lib64/libssl.so* + #fi + ``` + + 然后来到 `build-thirdparty.sh`，注释掉其他 `build_xxx`，仅打开 `build_zlib` 和 `build_openssl`，以及 `build_brpc` 和之后的 `build_xxx`。然后重新执行 `build-thirdparty.sh`。 + + * 编译到某个阶段卡住不动。 + + 不确定原因。解决方案：重跑 `build-thirdparty.sh`。`build-thirdparty.sh` 是可以重复执行的。 + +### 4. 编译Doris源码 + +执行 sh build.sh 即可。