ARROW-1123: Make jemalloc the default allocator

ci/travis_script_python.sh

@@ -106,10 +106,9 @@ python_version_tests() {
   # Other stuff pip install
   pip install -r requirements.txt
 
-  python setup.py build_ext --inplace --with-parquet --with-jemalloc
+  python setup.py build_ext --inplace --with-parquet
 
   python -c "import pyarrow.parquet"
-  python -c "import pyarrow._jemalloc"
 
   python -m pytest -vv -r sxX pyarrow --parquet
 

cpp/CMakeLists.txt

@@ -437,6 +437,14 @@ if (DEFINED ENV{GFLAGS_HOME})
   set(GFLAGS_HOME "$ENV{GFLAGS_HOME}")
 endif()
 
+# ----------------------------------------------------------------------
+# Find pthreads
+
+if (NOT MSVC)
+  find_library(PTHREAD_LIBRARY pthread)
+  message(STATUS "Found pthread: ${PTHREAD_LIBRARY}")
+endif()
+
 # ----------------------------------------------------------------------
 # Add Boost dependencies (code adapted from Apache Kudu (incubating))
 
@@ -756,7 +764,8 @@ if (ARROW_JEMALLOC)
   include_directories(SYSTEM ${JEMALLOC_INCLUDE_DIR})
   ADD_THIRDPARTY_LIB(jemalloc
     STATIC_LIB ${JEMALLOC_STATIC_LIB}
-    SHARED_LIB ${JEMALLOC_SHARED_LIB})
+    SHARED_LIB ${JEMALLOC_SHARED_LIB}
+    DEPS ${PTHREAD_LIBRARY})
 endif()
 
 ## Google PerfTools
@@ -803,20 +812,16 @@ include_directories(SYSTEM "${HADOOP_HOME}/include")
 # Linker setup
 ############################################################
 set(ARROW_MIN_TEST_LIBS
+  ${ARROW_STATIC_LINK_LIBS}
   arrow_static
   gtest
   gtest_main
   ${ARROW_BASE_LIBS})
 
-if (APPLE)
+if(NOT MSVC)
   set(ARROW_MIN_TEST_LIBS
     ${ARROW_MIN_TEST_LIBS}
     ${CMAKE_DL_LIBS})
-elseif(NOT MSVC)
-  set(ARROW_MIN_TEST_LIBS
-    ${ARROW_MIN_TEST_LIBS}
-    pthread
-    ${CMAKE_DL_LIBS})
 endif()
 
 set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS})
@@ -938,8 +943,55 @@ set(ARROW_STATIC_PRIVATE_LINK_LIBS
 if (NOT MSVC)
   set(ARROW_LINK_LIBS
     ${ARROW_LINK_LIBS}
-    ${CMAKE_DL_LIBS}
-    pthread)
+    ${CMAKE_DL_LIBS})
+endif()
+
+if (ARROW_JEMALLOC)
+  add_definitions(-DARROW_JEMALLOC)
+  # In the case that jemalloc is only available as a shared library also use it to
+  # link it in the static requirements. In contrast to other libraries we try in
+  # most cases to use the system provided version of jemalloc to better align with
+  # other potential users of jemalloc.
+  if (JEMALLOC_STATIC_LIB AND NOT ARROW_JEMALLOC_USE_SHARED)
+    set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_static)
+  else()
+    set(ARROW_JEMALLOC_STATIC_LINK_LIBS jemalloc_shared)
+  endif()
+
+  if (NOT APPLE)
+      set(ARROW_JEMALLOC_STATIC_LINK_LIBS ${ARROW_JEMALLOC_STATIC_LINK_LIBS} ${PTHREAD_LIBRARY} rt)
+  endif()
+
+  if (ARROW_JEMALLOC_USE_SHARED)
+    set(ARROW_JEMALLOC_SHARED_LINK_LIBS jemalloc_shared)
+  else()
+    if (CMAKE_COMPILER_IS_GNUCXX)
+      set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+        jemalloc_static
+        # For glibc <2.17 we need to link to librt.
+        # As we compile with --as-needed by default, the linker will omit this
+        # dependency if not required.
+        ${PTHREAD_LIBRARY}
+        rt
+        )
+    else()
+      set(ARROW_JEMALLOC_SHARED_LINK_LIBS
+        jemalloc_static
+        )
+    endif()
+  endif()
+  set(ARROW_SHARED_PRIVATE_LINK_LIBS
+    ${ARROW_SHARED_PRIVATE_LINK_LIBS}
+    ${ARROW_JEMALLOC_SHARED_LINK_LIBS})
+  set(ARROW_STATIC_LINK_LIBS
+    ${ARROW_STATIC_LINK_LIBS}
+    ${ARROW_JEMALLOC_STATIC_LINK_LIBS})
+elseif (NOT MSVC)
+  # We need to separate this as otherwise CMake would mess with the library
+  # linking order.
+  set(ARROW_LINK_LIBS
+    ${ARROW_LINK_LIBS}
+    ${PTHREAD_LIBRARY})
 endif()
 
 if(RAPIDJSON_VENDORED)
@@ -1021,10 +1073,6 @@ ADD_ARROW_LIB(arrow
 
 add_subdirectory(src/arrow/util)
 
-if(ARROW_JEMALLOC)
-  add_subdirectory(src/arrow/jemalloc)
-endif()
-
 if(ARROW_PYTHON)
   find_package(PythonLibsNew REQUIRED)
   find_package(NumPy REQUIRED)

cpp/cmake_modules/BuildUtils.cmake

@@ -46,6 +46,10 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
         set_target_properties(${AUG_LIB_NAME}
             PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     endif()
+    if(ARG_DEPS)
+      set_target_properties(${AUG_LIB_NAME}
+        PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+    endif()
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
   elseif(ARG_STATIC_LIB)
     add_library(${LIB_NAME} STATIC IMPORTED)
@@ -55,6 +59,10 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     add_library(${AUG_LIB_NAME} STATIC IMPORTED)
     set_target_properties(${AUG_LIB_NAME}
       PROPERTIES IMPORTED_LOCATION "${ARG_STATIC_LIB}")
+    if(ARG_DEPS)
+      set_target_properties(${AUG_LIB_NAME}
+        PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+    endif()
     message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
   elseif(ARG_SHARED_LIB)
     add_library(${LIB_NAME} SHARED IMPORTED)
@@ -72,14 +80,13 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
             PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     endif()
     message("Added shared library dependency ${LIB_NAME}: ${ARG_SHARED_LIB}")
+    if(ARG_DEPS)
+      set_target_properties(${AUG_LIB_NAME}
+        PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
+    endif()
   else()
     message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
   endif()
-
-  if(ARG_DEPS)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LINK_INTERFACE_LIBRARIES "${ARG_DEPS}")
-  endif()
 endfunction()
 
 function(ADD_ARROW_LIB LIB_NAME)

cpp/src/arrow/allocator-test.cc

@@ -33,7 +33,7 @@ TEST(stl_allocator, MemoryTracking) {
   ASSERT_EQ(0, pool->bytes_allocated());
 }
 
-#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER))
+#if !(defined(ARROW_VALGRIND) || defined(ADDRESS_SANITIZER) || defined(ARROW_JEMALLOC))
 
 TEST(stl_allocator, TestOOM) {
   stl_allocator<uint64_t> alloc;