ARROW-11460: [R] Use system libraries if present on Linux

ci/docker/linux-apt-r.dockerfile

@@ -100,7 +100,6 @@ ENV \
     ARROW_BUILD_STATIC=OFF \
     ARROW_BUILD_TESTS=OFF \
     ARROW_BUILD_UTILITIES=OFF \
-    ARROW_DEPENDENCY_SOURCE=SYSTEM \
     ARROW_FLIGHT=OFF \
     ARROW_GANDIVA=OFF \
     ARROW_NO_DEPRECATED_API=ON \

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -837,7 +837,17 @@ set(Boost_ADDITIONAL_VERSIONS
 # so we first need to determine whether we're building it
 if(ARROW_WITH_THRIFT AND Thrift_SOURCE STREQUAL "AUTO")
   find_package(Thrift 0.11.0 MODULE COMPONENTS ${ARROW_THRIFT_REQUIRED_COMPONENTS})
-  if(NOT Thrift_FOUND AND NOT THRIFT_FOUND)
+  if(Thrift_FOUND)
+    find_package(PkgConfig QUIET)
+    pkg_check_modules(THRIFT_PC
+                      thrift
+                      NO_CMAKE_PATH
+                      NO_CMAKE_ENVIRONMENT_PATH
+                      QUIET)
+    if(THRIFT_PC_FOUND)
+      string(APPEND ARROW_PC_REQUIRES_PRIVATE " thrift")
+    endif()
+  else()
     set(Thrift_SOURCE "BUNDLED")
   endif()
 endif()
@@ -1306,7 +1316,7 @@ endmacro()
 if(ARROW_WITH_THRIFT)
   # We already may have looked for Thrift earlier, when considering whether
   # to build Boost, so don't look again if already found.
-  if(NOT Thrift_FOUND AND NOT THRIFT_FOUND)
+  if(NOT Thrift_FOUND)
     # Thrift c++ code generated by 0.13 requires 0.11 or greater
     resolve_dependency(Thrift
                        REQUIRED_VERSION

dev/tasks/docker-tests/github.linux.yml

@@ -41,10 +41,11 @@ jobs:
         shell: bash
         run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }} {{ command|default("") }}
 
-    {% if '-r-' in image %}
+    {% if '-r' in image %}
       - name: Dump R install logs
         run: cat arrow/r/check/arrow.Rcheck/00install.out
         continue-on-error: true
+        if: always()
     {% endif %}
 
     {% if arrow.branch == 'master' %}

dev/tasks/tasks.yml

@@ -1013,6 +1013,26 @@ tasks:
   test-r-devdocs:
     ci: github
     template: r/github.devdocs.yml
+
+  test-r-depsource-auto:
+      ci: azure
+      template: r/azure.linux.yml
+      params:
+        r_org: rocker
+        r_image: r-base
+        r_tag: latest
+        flags: '-e ARROW_DEPENDENCY_SOURCE=AUTO'
+
+  test-r-depsource-system:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      env:
+        UBUNTU: 21.04
+        CLANG_TOOLS: 9 # can remove this when >=9 is the default
+      flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE -e ARROW_DEPENDENCY_SOURCE=SYSTEM'
+      image: ubuntu-r-only-r
+
 
 {% for r_org, r_image, r_tag in [("rhub", "ubuntu-gcc-release", "latest"),
                                  ("rocker", "r-base", "latest"),
@@ -1050,7 +1070,7 @@ tasks:
         UBUNTU: 21.04
         CLANG_TOOLS: 9 # can remove this when >=9 is the default
       flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE'
-      image: ubuntu-r
+      image: ubuntu-r-only-r
 
   # This also has -flto=auto
   test-r-gcc-11:

docker-compose.yml

@@ -1059,6 +1059,8 @@ services:
         /arrow/ci/scripts/r_test.sh /arrow"
 
   ubuntu-r-only-r:
+    environment:
+      ARROW_DEPENDENCY_SOURCE: ''
     extends: ubuntu-r
     command: >
       /bin/bash -c "

r/configure

@@ -78,13 +78,13 @@ elif [ "$INCLUDE_DIR" ] && [ "$LIB_DIR" ]; then
   PKG_CFLAGS="-I$INCLUDE_DIR $PKG_CFLAGS"
   PKG_DIRS="-L$LIB_DIR"
 else
-  # Use pkg-config if available and allowed
+  # Use pkg-config to find libarrow if available and allowed
   pkg-config --version >/dev/null 2>&1
   if [ $? -eq 0 ] && [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then
+    # Set the search paths and compile flags
     PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}`
-    PKGCONFIG_LIBS=`pkg-config --libs-only-l --silence-errors ${PKG_CONFIG_NAME}`
+    PKGCONFIG_LIBS=`pkg-config --libs-only-l --libs-only-other --silence-errors ${PKG_CONFIG_NAME}`
     PKGCONFIG_DIRS=`pkg-config --libs-only-L --silence-errors ${PKG_CONFIG_NAME}`
-    # TODO: what about --libs-only-other?
   fi
 
   if [ "$PKGCONFIG_CFLAGS" ] && [ "$PKGCONFIG_LIBS" ]; then
@@ -156,6 +156,19 @@ else
         fi
       fi
 
+      if [ "${ARROW_DEPENDENCY_SOURCE}" = "" ]; then
+        # TODO: BUNDLED is still default for now, but we plan to change it to AUTO
+        ARROW_DEPENDENCY_SOURCE=BUNDLED; export ARROW_DEPENDENCY_SOURCE
+      fi
+      if [ "${ARROW_DEPENDENCY_SOURCE}" = "AUTO" ]; then
+        pkg-config --version >/dev/null 2>&1
+        if [ $? -ne 0 ]; then
+          export ARROW_DEPENDENCY_SOURCE=BUNDLED
+          echo "**** Warning: ARROW_DEPENDENCY_SOURCE set to 'AUTO' but pkg-config not installed"
+          echo "**** ARROW_DEPENDENCY_SOURCE has been set to 'BUNDLED'"
+        fi
+      fi
+
       ${R_HOME}/bin/Rscript tools/nixlibs.R $VERSION
       PKG_CFLAGS="-I`pwd`/libarrow/arrow-${VERSION}/include $PKG_CFLAGS"
 
@@ -173,6 +186,11 @@ else
         BUNDLED_LIBS=`echo "$BUNDLED_LIBS" | sed -e "s/\\.a lib/ -l/g" | sed -e "s/\\.a$//" | sed -e "s/^lib/-l/" | tr '\n' ' ' | sed -e "s/ $//"`
         PKG_DIRS="-L`pwd`/$LIB_DIR"
 
+        # Use pkg-config to do static linking of libarrow's dependencies
+        if [ "$ARROW_DEPENDENCY_SOURCE" = "AUTO" ] || [ "$ARROW_DEPENDENCY_SOURCE" = "SYSTEM" ]; then
+          PKG_LIBS="$PKG_LIBS `PKG_CONFIG_PATH=${LIB_DIR}/pkgconfig pkg-config --libs-only-l --libs-only-other --static  --silence-errors ${PKG_CONFIG_NAME}`"
+        fi
+
         # When using brew's openssl it is not bundled and it is not on the system
         # search path  and so we must add the lib path to BUNDLED_LIBS if we are
         # using it. Note the order is important, this must be after the arrow

r/inst/build_arrow_static.sh

@@ -54,7 +54,8 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \
     -DARROW_COMPUTE=ON \
     -DARROW_CSV=ON \
     -DARROW_DATASET=${ARROW_DATASET:-ON} \
-    -DARROW_DEPENDENCY_SOURCE=BUNDLED \
+    -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-BUNDLED} \
+    -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \
     -DARROW_FILESYSTEM=ON \
     -DARROW_JEMALLOC=${ARROW_JEMALLOC:-$ARROW_DEFAULT_PARAM} \
     -DARROW_MIMALLOC=${ARROW_MIMALLOC:-ON} \