apache · bharathv · Aug 11, 2020 · Jul 31, 2020 · Aug 2, 2020
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -32,7 +32,6 @@ include(DownloadProject)
 include(ExecuteMaven)
 include(CheckCXXCompilerFlag)
 
-
 option(DOWNLOAD_DEPENDENCIES "Downloads and builds all dependencies locally " OFF)
 option(HBASE_TARGET_TAG "HBase tag to be used if HBASE_HOME is not set" "master")
 option(HBASE_HOME "Path to HBase" "")
@@ -56,12 +55,18 @@ endif()
 ######### Includes
 ## include the Protobuf generation code
 include(ProtobufGen)
+include(DownloadBoost)
 include(DownloadFolly)
 include(DownloadWangle)
 include(DownloadZookeeper)
 
 set(PROJECT_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/dependencies")
 
+set(BOOST_MIN_VERSION "1.6.1")
+# Union of all the libs needed by the current project and dependencies (like folly, wangle etc).
+# Just update this list if more libraries are needed and they will be included and linked automatically.
+set(BOOST_LIBS context coroutine thread system filesystem regex program_options)
+
 if (DOWNLOAD_DEPENDENCIES)
   ## we want to find the system protoc
   download_project(PROJ Protobuf PREFIX "${PROJECT_PREFIX}" IS_AUTOGEN GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git" GIT_TAG "3.5.1.1")
@@ -78,11 +83,11 @@ if (DOWNLOAD_DEPENDENCIES)
   set(PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_BINARY_DIR}/bin/protoc" CACHE STRING "" FORCE)
   ## Add CMAKE_MODULE_PATHS
 
+  list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local")
   list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/local")
   list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/protobuf/local")
   list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local")
   list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/wangle/local")
-
 else()
   list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/zookeeper/system")
   list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/system")
@@ -154,16 +159,16 @@ endif (OPENSSL_FOUND)
 
 
 if (DOWNLOAD_DEPENDENCIES)
+  download_boost(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} "${BOOST_LIBS}")
   download_folly(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
   download_wangle(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
   download_zookeeper(${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
 endif(DOWNLOAD_DEPENDENCIES)
 
-set(BOOST_MIN_VERSION "1.6.1")
 
 # ensure we have required dependencies
 find_package(Threads)
-find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS context thread system filesystem regex)
+find_package(Boost ${BOOST_MIN_VERSION} REQUIRED COMPONENTS "${BOOST_LIBS}")
 find_package(LibEvent REQUIRED)
 find_package(Gflags REQUIRED)
 if (DOWNLOAD_DEPENDENCIES)
@@ -309,6 +314,9 @@ add_custom_target(
     COMMAND ${CMAKE_SOURCE_DIR}/bin/cpplint.sh)
 
 if (DOWNLOAD_DEPENDENCIES)
+  add_dependencies(facebook-folly-proj boost)
+  add_dependencies(hbaseclient-static boost)
+  add_dependencies(hbaseclient-shared boost)
   add_dependencies(hbaseclient-static Protobuf)
   add_dependencies(hbaseclient-shared Protobuf)
   add_dependencies(facebook-wangle-proj facebook-folly-proj)
@@ -341,3 +349,4 @@ install(
     DIRECTORY "${CMAKE_BINARY_DIR_GEN}"
     DESTINATION include/hbase/if
     FILES_MATCHING PATTERN "hbase/if/*.h")
+
diff --git a/bin/hbase-native-client-personality.sh b/bin/hbase-native-client-personality.sh
@@ -73,7 +73,9 @@ function personality_globals
   # Passed to cmake command using a custom personality.
   CMAKE_ARGS="-DDOWNLOAD_DEPENDENCIES=ON"
   # Passed to make compilation command.
-  MAKE_COMPILE_ARGS="-j$(nproc)"
+  # Concurrency was determined after some trial and error. Higher
+  # values resulted in memory limit issues and compiler crashes.
+  MAKE_COMPILE_ARGS="-j4"
   # shellcheck disable=SC2034
   # Expected by Yetus for compiling non-jvm projects.
   JVM_REQUIRED=false
@@ -91,7 +93,10 @@ function personality_globals
   PROCLIMIT=10000
   # Override if you want to bump up the memlimit for docker.
   # shellcheck disable=SC2034
-  DOCKERMEMLIMIT=4g
+  DOCKERMEMLIMIT=16g
+  # shellcheck disable=SC2034
+  # Logs the ctest output when a test fails.
+  CTEST_OUTPUT_ON_FAILURE=1
 }
 
 ## @description  Queue up modules for this personality

diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake
@@ -53,7 +53,7 @@ function(createTests testName)
     #target_link_libraries(${testName} ${PROTOBUF_LIBRARY})
     #${PROTOBUF_LIBRARY}
 
-    target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS} 
+    target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_DL_LIBS} 
     ${Java_LIBRARIES}
     ${JNI_LIBRARIES}
     ${Boost_LIBRARIES}
@@ -63,7 +63,8 @@ function(createTests testName)
     ${GFLAGS_SHARED_LIB}
     ${KRB5_LIBRARIES}
     ${ZOOKEEPER_LIBRARIES} ${OPENSSL_LIBRARIES}
-    ${GLOG_SHARED_LIB})
+    ${GLOG_SHARED_LIB}
+    ${CMAKE_THREAD_LIBS_INIT})
 endfunction()
 enable_testing(test)
 SET(TEST_DIR ${CMAKE_SOURCE_DIR}/src/test)

diff --git a/cmake/DownloadBoost.cmake b/cmake/DownloadBoost.cmake
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+## Download Boost. 
+## SOURCE_DIR is typically the cmake source directory
+## BINARY_DIR is the build directory, typically 'build'
+## Sets BOOST_ROOT, search prefix for FindBoost.
+
+function(download_boost SOURCE_DIR BUILD_DIR BOOST_LIBS)
+  set(BOOST_DOWNLOAD_DIR "${BUILD_DIR}/dependencies/boost-download")
+  set(BOOST_SOURCE_DIR "${BUILD_DIR}/dependencies/boost-src")
+  set(BOOST_INSTALL_DIR "${BUILD_DIR}/dependencies/boost-install")
+
+  set(CFLAGS "-fPIC")
+  set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
+  # Only compile and install the needed libs.
+  set(LIBS_TO_COMPILE "")
+  foreach(lib ${BOOST_LIBS})
+    string(APPEND LIBS_TO_COMPILE --with-${lib} " ")
+  endforeach()
+
+  separate_arguments(BUILD_CMD UNIX_COMMAND
+    "./b2 cflags='${CFLAGS}' cxxflags='${CXXFLAGS}' variant=release link=static threading=multi ${LIBS_TO_COMPILE} install")
+
+  ExternalProject_Add(boost
+     URL "https://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz"
+     PREFIX "${BUILD_DIR}/dependencies"
+     DOWNLOAD_DIR ${BOOST_DOWNLOAD_DIR}
+     BUILD_IN_SOURCE true
+     SOURCE_DIR ${BOOST_SOURCE_DIR}
+     INSTALL_DIR ${BOOST_INSTALL_DIR}
+     CONFIGURE_COMMAND ./bootstrap.sh --prefix=${BOOST_INSTALL_DIR}
+     BUILD_COMMAND ${BUILD_CMD}
+     INSTALL_COMMAND ""
+  )
+  set(BOOST_ROOT ${BOOST_INSTALL_DIR} PARENT_SCOPE)
+endfunction(download_boost) 
diff --git a/cmake/DownloadFolly.cmake b/cmake/DownloadFolly.cmake
@@ -20,17 +20,28 @@
 ## BUILD_DIR is the build directory, typically 'build'
 
 function(download_folly SOURCE_DIR BUILD_DIR)
+
+  if (DOWNLOAD_DEPENDENCIES)
+    # Add custom boost include and lib paths.
+    set(CFLAGS "-fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+    set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -I${BOOST_ROOT}/include -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+    set(LDFLAGS "-L${BOOST_ROOT}/lib")
+    set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install
+      --with-boost-libdir=${BOOST_ROOT}/lib CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS} LDFLAGS=${LDFLAGS})
+  else()
+    set(CFLAGS "-fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+    set(CXXFLAGS "${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -l${CMAKE_DL_LIBS}")
+    set(CONFIGURE_CMD ./configure --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install CFLAGS=${CFLAGS} CXXFLAGS=${CXXFLAGS})
+  endif()
+
   ExternalProject_Add(
       facebook-folly-proj
       # TODO: Source version information from cmake file.
       URL "https://github.com/facebook/folly/archive/v2017.09.04.00.tar.gz"
       PREFIX "${BUILD_DIR}/dependencies"
       SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-src"
       BINARY_DIR ${BUILD_DIR}/dependencies/facebook-folly-proj-src/folly
-      CONFIGURE_COMMAND autoreconf -ivf
-      COMMAND ./configure  --prefix=${BUILD_DIR}/dependencies/facebook-folly-proj-install
-          "CFLAGS=-fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually
-  	   "CXXFLAGS=${CMAKE_CXX_FLAGS} -fPIC -lboost_context -lboost_coroutine -ldl" ## this version of folly does not support cmake so we must pass args manually
+      CONFIGURE_COMMAND autoreconf -ivf COMMAND ${CONFIGURE_CMD}
       UPDATE_COMMAND ""
   )
   set(FOLLY_ROOT_DIR "${BUILD_DIR}/dependencies/facebook-folly-proj-install" CACHE STRING "" FORCE)

diff --git a/cmake/DownloadProtobuf.cmake b/cmake/DownloadProtobuf.cmake
@@ -22,8 +22,6 @@
 #################### PROTOBUF
 
 function(download_protobuf SOURCE_DIR BINARY_DIR)
-
-
 	ExternalProject_Add(
 	  Protobuf
 	  GIT_REPOSITORY "https://github.com/protocolbuffers/protobuf.git"
@@ -46,7 +44,5 @@ function(download_protobuf SOURCE_DIR BINARY_DIR)
 	set(PROTOBUF_INCLUDE_DIRS "${CMAKE_CURRENT_BINARY_DIR}/dependencies/protobuf/include" CACHE STRING "" FORCE)
 	add_dependencies(protobuf Protobuf)
 	set(PROTOBUF_FOUND TRUE CACHE STRING "" FORCE)
-
-
 endfunction(download_protobuf)
 
diff --git a/cmake/DownloadWangle.cmake b/cmake/DownloadWangle.cmake
@@ -24,7 +24,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR)
   set(WANGLE_SOURCE_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-src")
   set(WANGLE_INSTALL_DIR "${BUILD_DIR}/dependencies/facebook-wangle-proj-install")
   if (DOWNLOAD_DEPENDENCIES)
-    set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake")
+    set(PATCH_FOLLY ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/cmake/folly/local/FindFolly.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cmake/boost/local/FindBoost.cmake" "${WANGLE_SOURCE_DIR}/wangle/cmake")
   else()
     set(PATCH_FOLLY "")
   endif() 
@@ -37,7 +37,7 @@ function(download_wangle SOURCE_DIR BUILD_DIR)
      SOURCE_DIR ${WANGLE_SOURCE_DIR}
      PATCH_COMMAND ${PATCH_FOLLY}
      INSTALL_DIR ${WANGLE_INSTALL_DIR}
-     CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source.
+     CONFIGURE_COMMAND ${CMAKE_COMMAND} -DBUILD_EXAMPLES=OFF -DCMAKE_CROSSCOMPILING=ON -DBUILD_TESTS=OFF -DFOLLY_ROOT_DIR=${FOLLY_ROOT_DIR} -DBOOST_ROOT=${BOOST_ROOT} -DCMAKE_INSTALL_PREFIX:PATH=${WANGLE_INSTALL_DIR} "${WANGLE_SOURCE_DIR}/wangle" # Tell CMake to use subdirectory as source.
   )
   set(WANGLE_ROOT_DIR ${WANGLE_INSTALL_DIR} CACHE STRING "" FORCE)
 endfunction(download_wangle) 
diff --git a/cmake/boost/local/FindBoost.cmake b/cmake/boost/local/FindBoost.cmake
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Stubs to allow us to find Boost libs
+
+set(Boost_INCLUDE_DIRS "${BOOST_ROOT}/include" CACHE STRING "" FORCE)
+set(Boost_INCLUDE_DIR "${BOOST_ROOT}/include" CACHE STRING "" FORCE)
+
+set(Boost_LIBRARIES "" CACHE STRING "" FORCE)
+foreach(COMPONENT ${Boost_FIND_COMPONENTS})
+  list(APPEND Boost_LIBRARIES "${BOOST_ROOT}/lib/${BYPRODUCT_PREFIX}boost_${COMPONENT}${BYPRODUCT_SUFFIX}")
+endforeach()
+
+set(Boost_FOUND "true" CACHE STRING "" FORCE)
+
+mark_as_advanced(
+    Boost_FOUND
+    Boost_INCLUDE_DIR
+    Boost_INCLUDE_DIRS
+    Boost_LIBRARIES
+)
+message("-- Boost found, ${Boost_LIBRARIES}")
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
@@ -30,7 +30,7 @@ ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
 RUN apt-get update && \
     apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format valgrind \
         wget libgflags-dev libgoogle-glog-dev dh-autoreconf pkg-config libssl-dev build-essential \
-        libboost-all-dev libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \
+        libevent-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip google-mock libsodium-dev libdouble-conversion-dev && \
     pip install yapf && \
     apt-get -qq clean && \
     apt-get -y -qq autoremove && \