From ac41cfc7e72221ae1ddedf3c6602154b86942704 Mon Sep 17 00:00:00 2001 From: Adonis Ling Date: Fri, 3 Mar 2023 14:18:02 +0800 Subject: [PATCH 1/5] [chore](macOS) Use LLVM Clang by default (#17292) Use LLVM Clang by default --- .github/workflows/be-ut-mac.yml | 1 + .github/workflows/build-thirdparty.yml | 1 + be/test/util/threadpool_test.cpp | 4 +++- docs/en/docs/install/source-install/compilation-mac.md | 7 +++---- docs/zh-CN/docs/install/source-install/compilation-mac.md | 5 ++--- env.sh | 1 + run-be-ut.sh | 2 +- 7 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index 3307ec11b8adef..12c73c220c0e62 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -79,6 +79,7 @@ jobs: 'openjdk@11' 'maven' 'node' + 'llvm' ) brew install "${cellars[@]}" diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 3f6e19dd13dbc8..d47cf0e04e09f3 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -89,6 +89,7 @@ jobs: 'openjdk-11-jdk' 'openjdk-11-jdk-headless' 'maven' + 'llvm' ) sudo apt update diff --git a/be/test/util/threadpool_test.cpp b/be/test/util/threadpool_test.cpp index 7ce870a6347b2b..f6ecaf3dc3e698 100644 --- a/be/test/util/threadpool_test.cpp +++ b/be/test/util/threadpool_test.cpp @@ -331,7 +331,9 @@ TEST_F(ThreadPoolTest, TestDeadlocks) { #ifdef __APPLE__ const char* death_msg = "_ZNSt3__1L8__invokeIRNS_6__bindIMN5doris10ThreadPoolEFvvEJPS3_EEEJEEEDTclscT_fp_" - "spscT0_fp0_EEOS9_DpOSA_"; + "spscT0_fp0_EEOS9_DpOSA_|_ZNSt3__18__invokeB6v15007IRNS_6__" + "bindIMN5doris10ThreadPoolEFvvEJPS3_EEEJEEEDTclclsr3stdE7declvalIT_" + "EEspclsr3stdE7declvalIT0_EEEEOS9_DpOSA_"; #else const char* death_msg = "_ZNSt5_BindIFMN5doris10ThreadPoolEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_" diff --git a/docs/en/docs/install/source-install/compilation-mac.md b/docs/en/docs/install/source-install/compilation-mac.md index 810d869e19e68f..fab174b155d8e7 100644 --- a/docs/en/docs/install/source-install/compilation-mac.md +++ b/docs/en/docs/install/source-install/compilation-mac.md @@ -30,16 +30,15 @@ This post introduces how to compile from source on macOS (both x86_64 and arm64) ## Prerequisites -1. macOS 12 (Monterey) or newer（_**both Intel chip and Apple Silicon chip are supported**_） -2. Apple Clang 13 or newer（the latest version is recommended） -3. [Homebrew](https://brew.sh/) +1. macOS 12 (Monterey) or newer（_**both Intel chip and Apple Silicon chips are supported**_） +2. [Homebrew](https://brew.sh/) ## Steps 1. Use [Homebrew](https://brew.sh/) to install tools ```shell brew install automake autoconf libtool pkg-config texinfo coreutils gnu-getopt \ - python cmake ninja ccache bison byacc gettext wget pcre maven openjdk@11 npm + python cmake ninja ccache bison byacc gettext wget pcre maven llvm openjdk@11 npm ``` 2. Compile from source diff --git a/docs/zh-CN/docs/install/source-install/compilation-mac.md b/docs/zh-CN/docs/install/source-install/compilation-mac.md index 3e679d355ab75c..2248b17293e476 100644 --- a/docs/zh-CN/docs/install/source-install/compilation-mac.md +++ b/docs/zh-CN/docs/install/source-install/compilation-mac.md @@ -31,15 +31,14 @@ under the License. ## 环境要求 1. macOS 12 (Monterey) 及以上（_**Intel和Apple Silicon均支持**_） -2. Apple Clang 13及以上（最好使用最新版本） -3. [Homebrew](https://brew.sh/) +2. [Homebrew](https://brew.sh/) ## 编译步骤 1. 使用[Homebrew](https://brew.sh/)安装依赖 ```shell brew install automake autoconf libtool pkg-config texinfo coreutils gnu-getopt \ - python cmake ninja ccache bison byacc gettext wget pcre maven openjdk@11 npm + python cmake ninja ccache bison byacc gettext wget pcre maven llvm openjdk@11 npm ``` 2. 编译源码 diff --git a/env.sh b/env.sh index 171a868468388e..64a4c94df66bfb 100755 --- a/env.sh +++ b/env.sh @@ -61,6 +61,7 @@ CELLARS=( wget pcre maven + llvm ) for cellar in "\${CELLARS[@]}"; do EXPORT_CELLARS="\${HOMEBREW_REPO_PREFIX}/opt/\${cellar}/bin:\${EXPORT_CELLARS}" diff --git a/run-be-ut.sh b/run-be-ut.sh index 91cf61148a0e9b..ad46a0080a3303 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -274,7 +274,7 @@ mkdir "${UT_TMP_DIR}" touch "${UT_TMP_DIR}/tmp_file" # set asan and ubsan env to generate core file -export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1 +export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=1 export UBSAN_OPTIONS=print_stacktrace=1 # find all executable test files From 12d6bc300dde738f0ca69a1ef1b8f66b273ccf21 Mon Sep 17 00:00:00 2001 From: Adonis Ling Date: Wed, 8 Mar 2023 10:21:45 +0800 Subject: [PATCH 2/5] [chore](macOS) Disable detect_container_overflow at BE startup (#17514) BE failed to start up due to container-overflow errors reported by address sanitizer. --- bin/start_be.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/start_be.sh b/bin/start_be.sh index 231986940ea03a..dcacbb7af36980 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -208,7 +208,7 @@ if [[ "${RUN_IN_AWS}" -eq 0 ]]; then fi ## set asan and ubsan env to generate core file -export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1 +export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=0 export UBSAN_OPTIONS=print_stacktrace=1 ## set TCMALLOC_HEAP_LIMIT_MB to limit memory used by tcmalloc From 9eddb53fec8544b08fb1d29d751e24149580b0c3 Mon Sep 17 00:00:00 2001 From: Adonis Ling Date: Wed, 22 Mar 2023 11:09:28 +0800 Subject: [PATCH 3/5] [chore](macOS) Specify the version of LLVM for Homebrew to install it (#17945) Clang 16 was released last week and we haven't ported the codebase to it. If Homebrew bumped the version of LLVM, our workflows would fail. --- .github/workflows/be-ut-mac.yml | 4 ++-- .github/workflows/build-thirdparty.yml | 4 ++-- docs/en/docs/install/source-install/compilation-mac.md | 2 +- docs/zh-CN/docs/install/source-install/compilation-mac.md | 2 +- env.sh | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index 12c73c220c0e62..5641cab7f8c126 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -67,7 +67,7 @@ jobs: 'texinfo' 'coreutils' 'gnu-getopt' - 'python' + 'python@3' 'cmake' 'ninja' 'ccache' @@ -79,7 +79,7 @@ jobs: 'openjdk@11' 'maven' 'node' - 'llvm' + 'llvm@15' ) brew install "${cellars[@]}" diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index d47cf0e04e09f3..5c4b8848e4b6c5 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -89,7 +89,6 @@ jobs: 'openjdk-11-jdk' 'openjdk-11-jdk-headless' 'maven' - 'llvm' ) sudo apt update @@ -135,7 +134,7 @@ jobs: 'texinfo' 'coreutils' 'gnu-getopt' - 'python' + 'python@3' 'cmake' 'ninja' 'ccache' @@ -147,6 +146,7 @@ jobs: 'openjdk@11' 'maven' 'node' + 'llvm@15' ) brew install "${packages[@]}" diff --git a/docs/en/docs/install/source-install/compilation-mac.md b/docs/en/docs/install/source-install/compilation-mac.md index fab174b155d8e7..88394c38d0496d 100644 --- a/docs/en/docs/install/source-install/compilation-mac.md +++ b/docs/en/docs/install/source-install/compilation-mac.md @@ -38,7 +38,7 @@ This post introduces how to compile from source on macOS (both x86_64 and arm64) 1. Use [Homebrew](https://brew.sh/) to install tools ```shell brew install automake autoconf libtool pkg-config texinfo coreutils gnu-getopt \ - python cmake ninja ccache bison byacc gettext wget pcre maven llvm openjdk@11 npm + python@3 cmake ninja ccache bison byacc gettext wget pcre maven llvm@15 openjdk@11 npm ``` 2. Compile from source diff --git a/docs/zh-CN/docs/install/source-install/compilation-mac.md b/docs/zh-CN/docs/install/source-install/compilation-mac.md index 2248b17293e476..a6393691ffe48f 100644 --- a/docs/zh-CN/docs/install/source-install/compilation-mac.md +++ b/docs/zh-CN/docs/install/source-install/compilation-mac.md @@ -38,7 +38,7 @@ under the License. 1. 使用[Homebrew](https://brew.sh/)安装依赖 ```shell brew install automake autoconf libtool pkg-config texinfo coreutils gnu-getopt \ - python cmake ninja ccache bison byacc gettext wget pcre maven llvm openjdk@11 npm + python@3 cmake ninja ccache bison byacc gettext wget pcre maven llvm@15 openjdk@11 npm ``` 2. 编译源码 diff --git a/env.sh b/env.sh index 64a4c94df66bfb..4d5f063f15d4fc 100755 --- a/env.sh +++ b/env.sh @@ -51,7 +51,7 @@ CELLARS=( texinfo coreutils gnu-getopt - python + python@3 cmake ninja ccache @@ -61,7 +61,7 @@ CELLARS=( wget pcre maven - llvm + llvm@15 ) for cellar in "\${CELLARS[@]}"; do EXPORT_CELLARS="\${HOMEBREW_REPO_PREFIX}/opt/\${cellar}/bin:\${EXPORT_CELLARS}" From a4ad2a8a1329f48560008333adf8829678aa95c2 Mon Sep 17 00:00:00 2001 From: Adonis Ling Date: Fri, 3 Mar 2023 23:34:54 +0800 Subject: [PATCH 4/5] [chore](workflow) Fix the workflow BE UT (macOS) (#17403) #17292 enabled detect_container_overflow which made the workflow BE UT (macOS) fail. --- run-be-ut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-be-ut.sh b/run-be-ut.sh index ad46a0080a3303..18e9d078b75d98 100755 --- a/run-be-ut.sh +++ b/run-be-ut.sh @@ -274,7 +274,7 @@ mkdir "${UT_TMP_DIR}" touch "${UT_TMP_DIR}/tmp_file" # set asan and ubsan env to generate core file -export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=1 +export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=0 export UBSAN_OPTIONS=print_stacktrace=1 # find all executable test files From b21db5d75b691f28adebf957aea1ffd657302c18 Mon Sep 17 00:00:00 2001 From: Adonis Ling Date: Mon, 24 Apr 2023 18:02:01 +0800 Subject: [PATCH 5/5] Fix BE UT --- be/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index b3ad8869104925..7e77a46a99323a 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -885,6 +885,9 @@ if (${MAKE_TEST} STREQUAL "ON") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcov") endif() add_definitions(-DBE_TEST) + if (ARCH_ARM) + add_compile_options(-ffp-contract=off) + endif() endif () add_subdirectory(${SRC_DIR}/agent)