diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 97029ce62ad..3a84a11c9b6 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -100,7 +100,6 @@ ENV \ ARROW_BUILD_STATIC=OFF \ ARROW_BUILD_TESTS=OFF \ ARROW_BUILD_UTILITIES=OFF \ - ARROW_DEPENDENCY_SOURCE=SYSTEM \ ARROW_FLIGHT=OFF \ ARROW_GANDIVA=OFF \ ARROW_NO_DEPRECATED_API=ON \ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 0631d277b08..bba1c2de1ba 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -837,7 +837,17 @@ set(Boost_ADDITIONAL_VERSIONS # so we first need to determine whether we're building it if(ARROW_WITH_THRIFT AND Thrift_SOURCE STREQUAL "AUTO") find_package(Thrift 0.11.0 MODULE COMPONENTS ${ARROW_THRIFT_REQUIRED_COMPONENTS}) - if(NOT Thrift_FOUND AND NOT THRIFT_FOUND) + if(Thrift_FOUND) + find_package(PkgConfig QUIET) + pkg_check_modules(THRIFT_PC + thrift + NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH + QUIET) + if(THRIFT_PC_FOUND) + string(APPEND ARROW_PC_REQUIRES_PRIVATE " thrift") + endif() + else() set(Thrift_SOURCE "BUNDLED") endif() endif() @@ -1306,7 +1316,7 @@ endmacro() if(ARROW_WITH_THRIFT) # We already may have looked for Thrift earlier, when considering whether # to build Boost, so don't look again if already found. - if(NOT Thrift_FOUND AND NOT THRIFT_FOUND) + if(NOT Thrift_FOUND) # Thrift c++ code generated by 0.13 requires 0.11 or greater resolve_dependency(Thrift REQUIRED_VERSION diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index dd347b8306c..2499cc43177 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -41,10 +41,11 @@ jobs: shell: bash run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }} {{ command|default("") }} - {% if '-r-' in image %} + {% if '-r' in image %} - name: Dump R install logs run: cat arrow/r/check/arrow.Rcheck/00install.out continue-on-error: true + if: always() {% endif %} {% if arrow.branch == 'master' %} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 70f28a9f6a1..b7160e2d97a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -1013,6 +1013,26 @@ tasks: test-r-devdocs: ci: github template: r/github.devdocs.yml + + test-r-depsource-auto: + ci: azure + template: r/azure.linux.yml + params: + r_org: rocker + r_image: r-base + r_tag: latest + flags: '-e ARROW_DEPENDENCY_SOURCE=AUTO' + + test-r-depsource-system: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 21.04 + CLANG_TOOLS: 9 # can remove this when >=9 is the default + flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE -e ARROW_DEPENDENCY_SOURCE=SYSTEM' + image: ubuntu-r-only-r + {% for r_org, r_image, r_tag in [("rhub", "ubuntu-gcc-release", "latest"), ("rocker", "r-base", "latest"), @@ -1050,7 +1070,7 @@ tasks: UBUNTU: 21.04 CLANG_TOOLS: 9 # can remove this when >=9 is the default flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE' - image: ubuntu-r + image: ubuntu-r-only-r # This also has -flto=auto test-r-gcc-11: diff --git a/docker-compose.yml b/docker-compose.yml index 4290578b9bd..cdbba66aebc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1059,6 +1059,8 @@ services: /arrow/ci/scripts/r_test.sh /arrow" ubuntu-r-only-r: + environment: + ARROW_DEPENDENCY_SOURCE: '' extends: ubuntu-r command: > /bin/bash -c " diff --git a/r/configure b/r/configure index 0972a99f8f3..d871028486a 100755 --- a/r/configure +++ b/r/configure @@ -78,13 +78,13 @@ elif [ "$INCLUDE_DIR" ] && [ "$LIB_DIR" ]; then PKG_CFLAGS="-I$INCLUDE_DIR $PKG_CFLAGS" PKG_DIRS="-L$LIB_DIR" else - # Use pkg-config if available and allowed + # Use pkg-config to find libarrow if available and allowed pkg-config --version >/dev/null 2>&1 if [ $? -eq 0 ] && [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then + # Set the search paths and compile flags PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}` - PKGCONFIG_LIBS=`pkg-config --libs-only-l --silence-errors ${PKG_CONFIG_NAME}` + PKGCONFIG_LIBS=`pkg-config --libs-only-l --libs-only-other --silence-errors ${PKG_CONFIG_NAME}` PKGCONFIG_DIRS=`pkg-config --libs-only-L --silence-errors ${PKG_CONFIG_NAME}` - # TODO: what about --libs-only-other? fi if [ "$PKGCONFIG_CFLAGS" ] && [ "$PKGCONFIG_LIBS" ]; then @@ -156,6 +156,19 @@ else fi fi + if [ "${ARROW_DEPENDENCY_SOURCE}" = "" ]; then + # TODO: BUNDLED is still default for now, but we plan to change it to AUTO + ARROW_DEPENDENCY_SOURCE=BUNDLED; export ARROW_DEPENDENCY_SOURCE + fi + if [ "${ARROW_DEPENDENCY_SOURCE}" = "AUTO" ]; then + pkg-config --version >/dev/null 2>&1 + if [ $? -ne 0 ]; then + export ARROW_DEPENDENCY_SOURCE=BUNDLED + echo "**** Warning: ARROW_DEPENDENCY_SOURCE set to 'AUTO' but pkg-config not installed" + echo "**** ARROW_DEPENDENCY_SOURCE has been set to 'BUNDLED'" + fi + fi + ${R_HOME}/bin/Rscript tools/nixlibs.R $VERSION PKG_CFLAGS="-I`pwd`/libarrow/arrow-${VERSION}/include $PKG_CFLAGS" @@ -173,6 +186,11 @@ else BUNDLED_LIBS=`echo "$BUNDLED_LIBS" | sed -e "s/\\.a lib/ -l/g" | sed -e "s/\\.a$//" | sed -e "s/^lib/-l/" | tr '\n' ' ' | sed -e "s/ $//"` PKG_DIRS="-L`pwd`/$LIB_DIR" + # Use pkg-config to do static linking of libarrow's dependencies + if [ "$ARROW_DEPENDENCY_SOURCE" = "AUTO" ] || [ "$ARROW_DEPENDENCY_SOURCE" = "SYSTEM" ]; then + PKG_LIBS="$PKG_LIBS `PKG_CONFIG_PATH=${LIB_DIR}/pkgconfig pkg-config --libs-only-l --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`" + fi + # When using brew's openssl it is not bundled and it is not on the system # search path and so we must add the lib path to BUNDLED_LIBS if we are # using it. Note the order is important, this must be after the arrow diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index fdd99ca52ef..2b2a475f1d2 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -54,7 +54,8 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_COMPUTE=ON \ -DARROW_CSV=ON \ -DARROW_DATASET=${ARROW_DATASET:-ON} \ - -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_DEPENDENCY_SOURCE=${ARROW_DEPENDENCY_SOURCE:-BUNDLED} \ + -DAWSSDK_SOURCE=${AWSSDK_SOURCE:-} \ -DARROW_FILESYSTEM=ON \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-$ARROW_DEFAULT_PARAM} \ -DARROW_MIMALLOC=${ARROW_MIMALLOC:-ON} \