From 121982c46e5e48ecd3ab1b265a31bf6b7af3774d Mon Sep 17 00:00:00 2001 From: Bharath Vissapragada Date: Thu, 30 Jul 2020 21:14:23 -0700 Subject: [PATCH 1/2] HBASE-24810: Download Boost with other dependencies Signed-off-by: Josh Elser --- CMakeLists.txt | 17 +++++++--- cmake/BuildTests.cmake | 5 +-- cmake/DownloadBoost.cmake | 52 +++++++++++++++++++++++++++++++ cmake/DownloadFolly.cmake | 19 ++++++++--- cmake/DownloadProtobuf.cmake | 4 --- cmake/DownloadWangle.cmake | 4 +-- cmake/boost/local/FindBoost.cmake | 36 +++++++++++++++++++++ docker-files/Dockerfile | 2 +- 8 files changed, 122 insertions(+), 17 deletions(-) create mode 100644 cmake/DownloadBoost.cmake create mode 100644 cmake/boost/local/FindBoost.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e845021..df07059 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,6 @@ include(DownloadProject) include(ExecuteMaven) include(CheckCXXCompilerFlag) - option(DOWNLOAD_DEPENDENCIES "Downloads and builds all dependencies locally " OFF) option(HBASE_TARGET_TAG "HBase tag to be used if HBASE_HOME is not set" "master") option(HBASE_HOME "Path to HBase" "") @@ -56,12 +55,18 @@ endif() ######### Includes ## include the Protobuf generation code include(ProtobufGen) +include(DownloadBoost) include(DownloadFolly) include(DownloadWangle) include(DownloadZookeeper) set(PROJECT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/dependencies") +set(BOOST_MIN_VERSION "1.6.1") +# Union of all the libs needed by the current project and dependencies (like folly, wangle etc). +# Just update this list if more libraries are needed and they will be included and linked automatically. +set(BOOST_LIBS context coroutine thread system filesystem regex program_options) + if (DOWNLOAD_DEPENDENCIES) ## we want to find the system protoc download_project(PROJ Protobuf PREFIX "${PROJECT_PREFIX}" IS_AUTOGEN GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git" GIT_TAG "3.5.1.1") @@ -78,11 +83,11 @@ if (DOWNLOAD_DEPENDENCIES) set(PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_BINARY_DIR}/bin/protoc" CACHE STRING "" FORCE) ## Add CMAKE_MODULE_PATHS + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/local") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf/local") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/wangle/local") - else() list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/system") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/system") @@ -154,16 +159,16 @@ endif (OPENSSL_FOUND) if (DOWNLOAD_DEPENDENCIES) + download_boost(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} "${BOOST_LIBS}") download_folly(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) download_wangle(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) download_zookeeper(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) endif(DOWNLOAD_DEPENDENCIES) -set(BOOST_MIN_VERSION "1.6.1") # ensure we have required dependencies find_package(Threads) -find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS context thread system filesystem regex) +find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS "${BOOST_LIBS}") find_package(LibEvent REQUIRED) find_package(Gflags REQUIRED) if (DOWNLOAD_DEPENDENCIES) @@ -309,6 +314,9 @@ add_custom_target( COMMAND ${CMAKE_SOURCE_DIR}/bin/cpplint.sh) if (DOWNLOAD_DEPENDENCIES) + add_dependencies(facebook-folly-proj boost) + add_dependencies(hbaseclient-static boost) + add_dependencies(hbaseclient-shared boost) add_dependencies(hbaseclient-static Protobuf) add_dependencies(hbaseclient-shared Protobuf) add_dependencies(facebook-wangle-proj facebook-folly-proj) @@ -341,3 +349,4 @@ install( DIRECTORY "${CMAKE_BINARY_DIR_GEN}" DESTINATION include/hbase/if FILES_MATCHING PATTERN "hbase/if/*.h") + diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake index e7b4243..4808fc3 100644 --- a/cmake/BuildTests.cmake +++ b/cmake/BuildTests.cmake @@ -53,7 +53,7 @@ function(createTests testName) #target_link_libraries(${testName} ${PROTOBUF_LIBRARY}) #${PROTOBUF_LIBRARY} - target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS} + target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_DL_LIBS} ${Java_LIBRARIES} ${JNI_LIBRARIES} ${Boost_LIBRARIES} @@ -63,7 +63,8 @@ function(createTests testName) ${GFLAGS_SHARED_LIB} ${KRB5_LIBRARIES} ${ZOOKEEPER_LIBRARIES} ${OPENSSL_LIBRARIES} - ${GLOG_SHARED_LIB}) + ${GLOG_SHARED_LIB} + ${CMAKE_THREAD_LIBS_INIT}) endfunction() enable_testing(test) SET(TEST_DIR ${CMAKE_SOURCE_DIR}/src/test) diff --git a/cmake/DownloadBoost.cmake b/cmake/DownloadBoost.cmake new file mode 100644 index 0000000..388a511 --- /dev/null +++ b/cmake/DownloadBoost.cmake @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +## Download Boost. +## SOURCE_DIR is typically the cmake source directory +## BINARY_DIR is the build directory, typically 'build' +## Sets BOOST_ROOT, search prefix for FindBoost. + +function(download_boost SOURCE_DIR BUILD_DIR BOOST_LIBS) + set(BOOST_DOWNLOAD_DIR "${BUILD_DIR}/dependencies/boost-download") + set(BOOST_SOURCE_DIR "${BUILD_DIR}/dependencies/boost-src") + set(BOOST_INSTALL_DIR "${BUILD_DIR}/dependencies/boost-install") + + set(CFLAGS "-fPIC") + set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC") + + # Only compile and install the needed libs. + set(LIBS_TO_COMPILE "") + foreach(lib ${BOOST_LIBS}) + string(APPEND LIBS_TO_COMPILE --with-${lib} " ") + endforeach() + + separate_arguments(BUILD_CMD UNIX_COMMAND + "./b2 cflags='${CFLAGS}' cxxflags='${CXXFLAGS}' variant=release link=static threading=multi ${LIBS_TO_COMPILE} install") + + ExternalProject_Add(boost + URL "https://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz" + PREFIX "${BUILD_DIR}/dependencies" + DOWNLOAD_DIR ${BOOST_DOWNLOAD_DIR} + BUILD_IN_SOURCE true + SOURCE_DIR ${BOOST_SOURCE_DIR} + INSTALL_DIR ${BOOST_INSTALL_DIR} + CONFIGURE_COMMAND ./bootstrap.sh --prefix=${BOOST_INSTALL_DIR} + BUILD_COMMAND ${BUILD_CMD} + INSTALL_COMMAND "" + ) + set(BOOST_ROOT ${BOOST_INSTALL_DIR} PARENT_SCOPE) +endfunction(download_boost) diff --git a/cmake/DownloadFolly.cmake b/cmake/DownloadFolly.cmake index 16f479b..1ff5556 100644 --- a/cmake/DownloadFolly.cmake +++ b/cmake/DownloadFolly.cmake @@ -20,6 +20,20 @@ ## BUILD_DIR is the build directory, typically 'build' function(download_folly SOURCE_DIR BUILD_DIR) + + if (DOWNLOAD_DEPENDENCIES) + # Add custom boost include and lib paths. + set(CFLAGS "-fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}") + set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}") + set(LDFLAGS "-L${BOOST_ROOT}/lib") + set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install + --with-boost-libdir=${BOOST_ROOT}/lib CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS} LDFLAGS=${LDFLAGS}) + else() + set(CFLAGS "-fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}") + set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}") + set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS}) + endif() + ExternalProject_Add( facebook-folly-proj # TODO: Source version information from cmake file. @@ -27,10 +41,7 @@ function(download_folly SOURCE_DIR BUILD_DIR) PREFIX "${BUILD_DIR}/dependencies" SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-src" BINARY_DIR ${BUILD_DIR}/dependencies/facebook-folly-proj-src/folly - CONFIGURE_COMMAND autoreconf -ivf - COMMAND ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install - "CFLAGS=-fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually - "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually + CONFIGURE_COMMAND autoreconf -ivf COMMAND ${CONFIGURE_CMD} UPDATE_COMMAND "" ) set(FOLLY_ROOT_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-install" CACHE STRING "" FORCE) diff --git a/cmake/DownloadProtobuf.cmake b/cmake/DownloadProtobuf.cmake index ca8260c..09e7c67 100644 --- a/cmake/DownloadProtobuf.cmake +++ b/cmake/DownloadProtobuf.cmake @@ -22,8 +22,6 @@ #################### PROTOBUF function(download_protobuf SOURCE_DIR BINARY_DIR) - - ExternalProject_Add( Protobuf GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git" @@ -46,7 +44,5 @@ function(download_protobuf SOURCE_DIR BINARY_DIR) set(PROTOBUF_INCLUDE_DIRS "${CMAKE_CURRENT_BINARY_DIR}/dependencies/protobuf/include" CACHE STRING "" FORCE) add_dependencies(protobuf Protobuf) set(PROTOBUF_FOUND TRUE CACHE STRING "" FORCE) - - endfunction(download_protobuf) diff --git a/cmake/DownloadWangle.cmake b/cmake/DownloadWangle.cmake index 1bfff5f..74c97a6 100644 --- a/cmake/DownloadWangle.cmake +++ b/cmake/DownloadWangle.cmake @@ -24,7 +24,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR) set(WANGLE_SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-src") set(WANGLE_INSTALL_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-install") if (DOWNLOAD_DEPENDENCIES) - set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake") + set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local/FindBoost.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake") else() set(PATCH_FOLLY "") endif() @@ -37,7 +37,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR) SOURCE_DIR ${WANGLE_SOURCE_DIR} PATCH_COMMAND ${PATCH_FOLLY} INSTALL_DIR ${WANGLE_INSTALL_DIR} - CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source. + CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DBOOST_ROOT=${BOOST_ROOT} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source. ) set(WANGLE_ROOT_DIR ${WANGLE_INSTALL_DIR} CACHE STRING "" FORCE) endfunction(download_wangle) diff --git a/cmake/boost/local/FindBoost.cmake b/cmake/boost/local/FindBoost.cmake new file mode 100644 index 0000000..60d7ee0 --- /dev/null +++ b/cmake/boost/local/FindBoost.cmake @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Stubs to allow us to find Boost libs + +set(Boost_INCLUDE_DIRS "${BOOST_ROOT}/include" CACHE STRING "" FORCE) +set(Boost_INCLUDE_DIR "${BOOST_ROOT}/include" CACHE STRING "" FORCE) + +set(Boost_LIBRARIES "" CACHE STRING "" FORCE) +foreach(COMPONENT ${Boost_FIND_COMPONENTS}) + list(APPEND Boost_LIBRARIES "${BOOST_ROOT}/lib/${BYPRODUCT_PREFIX}boost_${COMPONENT}${BYPRODUCT_SUFFIX}") +endforeach() + +set(Boost_FOUND "true" CACHE STRING "" FORCE) + +mark_as_advanced( + Boost_FOUND + Boost_INCLUDE_DIR + Boost_INCLUDE_DIRS + Boost_LIBRARIES +) +message("-- Boost found, ${Boost_LIBRARIES}") diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile index 3cad6d2..82545d7 100644 --- a/docker-files/Dockerfile +++ b/docker-files/Dockerfile @@ -30,7 +30,7 @@ ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/" RUN apt-get update && \ apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format valgrind \ wget libgflags-dev libgoogle-glog-dev dh-autoreconf pkg-config libssl-dev build-essential \ - libboost-all-dev libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \ + libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \ pip install yapf && \ apt-get -qq clean && \ apt-get -y -qq autoremove && \ From b04cd9a78dd526bdf85f0e8c077451a247dedb86 Mon Sep 17 00:00:00 2001 From: Bharath Vissapragada Date: Sun, 2 Aug 2020 13:42:58 -0700 Subject: [PATCH 2/2] HBASE-24642 (addendum): Reduce make concurrency and increase container mem limit Signed-off-by: Josh Elser --- bin/hbase-native-client-personality.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/hbase-native-client-personality.sh b/bin/hbase-native-client-personality.sh index af6ee3f..b3bcf33 100755 --- a/bin/hbase-native-client-personality.sh +++ b/bin/hbase-native-client-personality.sh @@ -73,7 +73,9 @@ function personality_globals # Passed to cmake command using a custom personality. CMAKE_ARGS="-DDOWNLOAD_DEPENDENCIES=ON" # Passed to make compilation command. - MAKE_COMPILE_ARGS="-j$(nproc)" + # Concurrency was determined after some trial and error. Higher + # values resulted in memory limit issues and compiler crashes. + MAKE_COMPILE_ARGS="-j4" # shellcheck disable=SC2034 # Expected by Yetus for compiling non-jvm projects. JVM_REQUIRED=false @@ -91,7 +93,10 @@ function personality_globals PROCLIMIT=10000 # Override if you want to bump up the memlimit for docker. # shellcheck disable=SC2034 - DOCKERMEMLIMIT=4g + DOCKERMEMLIMIT=16g + # shellcheck disable=SC2034 + # Logs the ctest output when a test fails. + CTEST_OUTPUT_ON_FAILURE=1 } ## @description Queue up modules for this personality