ARROW-6793: [R] Arrow C++ binary packaging for Linux

.env

@@ -30,11 +30,14 @@ GO=1.12
 NODE=11
 MAVEN=3.5.4
 JDK=8
-R=3.6.1
 PANDAS=latest
 DASK=latest
 TURBODBC=latest
 HDFS=2.9.2
 SPARK=master
 DOTNET=2.1
 R=3.6
+# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
+R_ORG=rhub
+R_IMAGE=ubuntu-gcc-release
+R_TAG=latest

.github/workflows/dev_cron.yml

@@ -18,6 +18,12 @@
 name: Dev Cron
 
 on:
+  push:
+    paths:
+      - '.github/workflows/dev_cron.yml'
+  pull_request:
+    paths:
+      - '.github/workflows/dev_cron.yml'
   schedule:
     - cron: |
         */15 * * * *

.github/workflows/r.yml

@@ -32,7 +32,6 @@ on:
       - 'r/**'
 
 jobs:
-
   ubuntu:
     name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }}
     runs-on: ubuntu-latest
@@ -71,39 +70,37 @@ jobs:
                        -p ${{ secrets.DOCKERHUB_TOKEN }}
           docker-compose push ubuntu-r
 
-  conda:
-    name: AMD64 Conda R ${{ matrix.r }}
+  rstudio:
+    name: "rstudio/r-base:${{ matrix.r_version }}-${{ matrix.r_image }}"
     runs-on: ubuntu-latest
-    if: github.event_name == 'push'
     strategy:
       fail-fast: false
       matrix:
-        r: [3.6]
+        # See https://hub.docker.com/r/rstudio/r-base
+        r_version: ["3.6"]
+        r_image:
+          - centos7
     env:
-      R: ${{ matrix.r }}
+      R_ORG: "rstudio"
+      R_IMAGE: "r-base"
+      R_TAG: ${{ matrix.r_version }}-${{ matrix.r_image }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v1
-        with:
-          submodules: true
       - name: Docker Pull
         shell: bash
-        run: |
-          docker-compose pull --ignore-pull-failures conda-cpp
-          docker-compose pull --ignore-pull-failures conda-r
+        run: docker-compose pull --ignore-pull-failures r
       - name: Docker Build
         shell: bash
-        run: |
-          docker-compose build conda-cpp
-          docker-compose build conda-r
+        run: docker-compose build r
       - name: Docker Run
         shell: bash
-        run: docker-compose run conda-r
+        run: docker-compose run r
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
         continue-on-error: true
         shell: bash
         run: |
           docker login -u ${{ secrets.DOCKERHUB_USER }} \
                        -p ${{ secrets.DOCKERHUB_TOKEN }}
-          docker-compose push conda-r
+          docker-compose push r

ci/docker/linux-apt-r.dockerfile

@@ -54,16 +54,10 @@ RUN apt-get update -y && \
 
 # Ensure parallel R package installation, set CRAN repo mirror,
 # and use pre-built binaries where possible
-RUN printf "\
-    options(Ncpus = parallel::detectCores(), \
-            repos = 'https://demo.rstudiopm.com/all/__linux__/"$(lsb_release -cs)"/latest', \
-            HTTPUserAgent = sprintf(\
-                'R/%%s R (%%s)', getRversion(), \
-                paste(getRversion(), R.version\$platform, R.version\$arch, R.version\$os)))\n" \
-    >> /etc/R/Rprofile.site
-
+COPY ci/etc/rprofile /arrow/ci/etc/
+RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site
 # Also ensure parallel compilation of C/C++ code
-RUN echo "MAKEFLAGS=-j$(R --slave -e 'cat(parallel::detectCores())')" >> /usr/lib/R/etc/Makeconf
+RUN echo "MAKEFLAGS=-j$(R --slave -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf
 
 COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
 COPY r/DESCRIPTION /arrow/r/

ci/docker/linux-r.dockerfile

@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# General purpose Dockerfile to take a Docker image containing R
+# and install Arrow R package dependencies
+
+ARG base
+FROM ${base}
+
+# Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile)
+ENV PATH "${RPREFIX}/bin:${PATH}"
+# Ensure parallel R package installation, set CRAN repo mirror,
+# and use pre-built binaries where possible
+COPY ci/etc/rprofile /arrow/ci/etc/
+RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site
+# Also ensure parallel compilation of C/C++ code
+RUN echo "MAKEFLAGS=-j$(R --slave -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Makeconf
+# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
+RUN Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}'
+
+COPY ci/scripts/r_deps.sh /arrow/ci/scripts/
+COPY r/DESCRIPTION /arrow/r/
+RUN /arrow/ci/scripts/r_deps.sh /arrow

ci/etc/rprofile

@@ -0,0 +1,51 @@
+.pick_cran <- function() {
+  # Return a CRAN repo URL, preferring RSPM binaries if available for this OS
+  rspm_template <- "https://demo.rstudiopm.com/all/__linux__/%s/latest"
+  supported_os <- c("xenial", "bionic", "centos7", "opensuse42", "opensuse15")
+
+  if (nzchar(Sys.which("lsb_release"))) {
+    os <- tolower(system("lsb_release -cs", intern = TRUE))
+    if (os %in% supported_os) {
+      return(sprintf(rspm_template, os))
+    }
+  }
+  if (file.exists("/etc/os-release")) {
+    os_release <- readLines("/etc/os-release")
+    vals <- sub("^.*=(.*)$", "\\1", os_release)
+    os <- intersect(vals, supported_os)
+    if (length(os)) {
+      # e.g. "bionic"
+      return(sprintf(rspm_template, os))
+    } else {
+      names(vals) <- sub("^(.*)=.*$", "\\1", os_release)
+      if (vals["ID"] == "opensuse") {
+        version <- sub('^"?([0-9]+).*"?.*$', "\\1", vals["VERSION_ID"])
+        os <- paste0("opensuse", version)
+        if (os %in% supported_os) {
+          return(sprintf(rspm_template, os))
+        }
+      }
+    }
+  }
+  if (file.exists("/etc/system-release")) {
+    # Something like "CentOS Linux release 7.7.1908 (Core)"
+    system_release <- tolower(utils::head(readLines("/etc/system-release"), 1))
+    # Extract from that the distro and the major version number
+    os <- sub("^([a-z]+) .* ([0-9]+).*$", "\\1\\2", system_release)
+    if (os %in% supported_os) {
+      return(sprintf(rspm_template, os))
+    }
+  }
+
+  return("https://cloud.r-project.org")
+}
+
+options(
+  Ncpus = parallel::detectCores(),
+  repos = tryCatch(.pick_cran(), error = function(e) "https://cloud.r-project.org"),
+  HTTPUserAgent = sprintf(
+    'R/%s R (%s)',
+    getRversion(),
+    paste(getRversion(), R.version$platform, R.version$arch, R.version$os)
+  )
+)

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -1038,20 +1038,25 @@ macro(build_thrift)
 
   # Thrift also uses boost. Forward important boost settings if there were ones passed.
   if(DEFINED BOOST_ROOT)
-    set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBOOST_ROOT=${BOOST_ROOT}")
+    list(APPEND THRIFT_CMAKE_ARGS "-DBOOST_ROOT=${BOOST_ROOT}")
   endif()
   if(DEFINED Boost_NAMESPACE)
-    set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DBoost_NAMESPACE=${Boost_NAMESPACE}")
+    list(APPEND THRIFT_CMAKE_ARGS "-DBoost_NAMESPACE=${Boost_NAMESPACE}")
+  endif()
+
+  if(DEFINED FLEX_ROOT)
+    # thrift hasn't set the cmake policy that lets us use _ROOT, so work around
+    list(APPEND THRIFT_CMAKE_ARGS "-DFLEX_EXECUTABLE=${FLEX_ROOT}/flex")
   endif()
 
   set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift")
   if(MSVC)
     if(ARROW_USE_STATIC_CRT)
       set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}mt")
-      set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON")
+      list(APPEND THRIFT_CMAKE_ARGS "-DWITH_MT=ON")
     else()
       set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}md")
-      set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF")
+      list(APPEND THRIFT_CMAKE_ARGS "-DWITH_MT=OFF")
     endif()
   endif()
   if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")

dev/release/rat_exclude_files.txt

@@ -8,6 +8,7 @@
 *.json
 *.snap
 .github/ISSUE_TEMPLATE.md
+ci/etc/rprofile
 cpp/CHANGELOG_PARQUET.md
 cpp/src/arrow/io/mman.h
 cpp/src/arrow/util/random.h

dev/tasks/r/azure.linux.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+jobs:
+- job: linux
+  pool:
+    vmImage: ubuntu-latest
+  timeoutInMinutes: 360
+  steps:
+    - script: |
+        set -ex
+        git clone --no-checkout {{ arrow.remote }} arrow
+        git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
+        git -C arrow checkout FETCH_HEAD
+        git -C arrow submodule update --init --recursive
+      displayName: Clone arrow
+
+    - script: |
+        set -ex
+        docker -v
+        docker-compose -v
+        cd arrow
+        R_ORG={{ r_org }}
+        R_IMAGE={{ r_image }}
+        R_TAG={{ r_tag }}
+        docker-compose pull --ignore-pull-failures r
+        docker-compose build r
+      displayName: Docker build
+
+    - script: |
+        set -ex
+        cd arrow
+        R_ORG={{ r_org }}
+        R_IMAGE={{ r_image }}
+        R_TAG={{ r_tag }}
+        docker-compose run r
+      displayName: Docker run