ARROW-17610: [C++] Support additional source types in SourceNode

.env

@@ -58,7 +58,7 @@ CUDA=11.0.3
 DASK=latest
 DOTNET=6.0
 GCC_VERSION=""
-GO=1.16
+GO=1.17
 STATICCHECK=v0.2.2
 HDFS=3.2.1
 JDK=8
@@ -86,8 +86,8 @@ ARROW_R_DEV=TRUE
 R_PRUNE_DEPS=FALSE
 TZ=UTC
 
-# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
-DEVTOOLSET_VERSION=-1
+# Any non-empty string will install devtoolset-${DEVTOOLSET_VERSION}
+DEVTOOLSET_VERSION=
 
 # Used through docker-compose.yml and serves as the default version for the
 # ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the

.github/workflows/comment_bot.yml

@@ -66,8 +66,9 @@ jobs:
           DEFAULT_BRANCH=${{ github.event.repository.default_branch }}
           git remote add upstream https://github.com/apache/arrow
           git fetch upstream
+
           changed() {
-            git diff --name-only HEAD..upstream/$DEFAULT_BRANCH | grep -e "$1" >/dev/null 2>&1
+            git diff --name-only upstream/$DEFAULT_BRANCH... | grep -e "$1" >/dev/null 2>&1
           }
           if changed '^r/.*\.R$'; then
             echo "R_DOCS=true" >> $GITHUB_ENV
@@ -82,7 +83,7 @@ jobs:
           if changed '^r/src'; then
             echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV
           fi
-      - name: Ensure clang-format has the appropriate versoin
+      - name: Ensure clang-format has the appropriate version
         if: env.CMAKE_FORMAT == 'true' ||
           env.CLANG_FORMAT_CPP == 'true' ||
           env.CLANG_FORMAT_R == 'true' ||
@@ -124,18 +125,16 @@ jobs:
         run: |
           source("ci/etc/rprofile")
           install.packages(c("remotes", "roxygen2"))
-          # We currently need dev roxygen2 (> 7.1.1) until they release
-          remotes::install_github("r-lib/roxygen2")
           remotes::install_deps("r")
           roxygen2::roxygenize("r")
       - name: Style R code
         if: env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything')
         shell: Rscript {0}
         run: |
-          changed_files <- system("git diff --name-only HEAD..upstream/${{ github.event.repository.default_branch }} 2>&1", intern = TRUE)
+          changed_files <- system("git diff --name-only upstream/${{ github.event.repository.default_branch }}... 2>&1", intern = TRUE)
           # only grab the .R files under r/
           changed_files <- grep('^r/.*\\.R$', changed_files, value = TRUE)
-          # remove latin1 which is unstylable due to encoding and codegen.R which is unique
+          # remove codegen.R and other possible exclusions
           changed_files <- changed_files[!changed_files %in% file.path("r", source("r/.styler_excludes.R")$value)]
           source("ci/etc/rprofile")
           install.packages(c("remotes", "styler"))

.github/workflows/cpp.yml

@@ -56,7 +56,7 @@ jobs:
     name: ${{ matrix.title }}
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 60
+    timeout-minutes: 75
     strategy:
       fail-fast: false
       matrix:
@@ -122,7 +122,7 @@ jobs:
     name: AMD64 macOS 11 C++
     runs-on: macos-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
-    timeout-minutes: 60
+    timeout-minutes: 75
     strategy:
       fail-fast: false
     env:
@@ -146,8 +146,7 @@ jobs:
       ARROW_WITH_SNAPPY: ON
       ARROW_WITH_ZLIB: ON
       ARROW_WITH_ZSTD: ON
-      # System Abseil installed by Homebrew uses C++ 17
-      CMAKE_CXX_STANDARD: 17
+      GTest_SOURCE: BUNDLED
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v3
@@ -180,7 +179,11 @@ jobs:
           key: cpp-ccache-macos-${{ hashFiles('cpp/**') }}
           restore-keys: cpp-ccache-macos-
       - name: Build
-        run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
+        # use brew version of clang, to be consistent with LLVM lib, see ARROW-17790.
+        run: |
+          export CC=$(brew --prefix llvm)/bin/clang
+          export CXX=$(brew --prefix llvm)/bin/clang++
+          ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
       - name: Test
         shell: bash
         run: |
@@ -257,7 +260,7 @@ jobs:
       - name: Install ccache
         shell: bash
         run: |
-          ci/scripts/install_ccache.sh 4.6.2 /usr
+          ci/scripts/install_ccache.sh 4.6.3 /usr
       - name: Setup ccache
         shell: bash
         run: |
@@ -271,8 +274,11 @@ jobs:
         uses: actions/cache@v2
         with:
           path: ${{ steps.ccache-info.outputs.cache-dir }}
-          key: cpp-ccache-windows-${{ hashFiles('cpp/**') }}
-          restore-keys: cpp-ccache-windows-
+          key: cpp-ccache-windows-${{ env.CACHE_VERSION }}-${{ hashFiles('cpp/**') }}
+          restore-keys: cpp-ccache-windows-${{ env.CACHE_VERSION }}-
+        env:
+          # We can invalidate the current cache by updating this.
+          CACHE_VERSION: "2022-09-13"
       - name: Build
         shell: cmd
         run: |

.github/workflows/go.yml

@@ -50,9 +50,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.16, 1.18]
+        go: [1.17, 1.18]
         include:
-          - go: 1.16
+          - go: 1.17
             staticcheck: v0.2.2
           - go: 1.18
             staticcheck: latest
@@ -86,9 +86,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.16, 1.18]
+        go: [1.17, 1.18]
         include:
-          - go: 1.16
+          - go: 1.17
             staticcheck: v0.2.2
           - go: 1.18
             staticcheck: latest
@@ -123,9 +123,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.16, 1.18]
+        go: [1.17, 1.18]
         include:
-          - go: 1.16
+          - go: 1.17
             staticcheck: v0.2.2
           - go: 1.18
             staticcheck: latest
@@ -158,9 +158,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.16, 1.18]
+        go: [1.17, 1.18]
         include:
-          - go: 1.16
+          - go: 1.17
             staticcheck: v0.2.2
           - go: 1.18
             staticcheck: latest
@@ -193,9 +193,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.16, 1.18]
+        go: [1.17, 1.18]
         include:
-          - go: 1.16
+          - go: 1.17
             staticcheck: v0.2.2
           - go: 1.18
             staticcheck: latest
@@ -228,9 +228,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go: [1.16, 1.18]
+        go: [1.17, 1.18]
         include:
-          - go: 1.16
+          - go: 1.17
             staticcheck: v0.2.2
           - go: 1.18
             staticcheck: latest

.github/workflows/java_nightly.yml

@@ -97,15 +97,19 @@ jobs:
       - shell: bash
         name: Build Repository
         run: |
+          DATE=$(date +%Y-%m-%d)
           if [ -z $PREFIX ]; then
-            PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0
+            PREFIX=nightly-packaging-${DATE}-0
           fi
-          PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9].[0-9].[0-9].dev[0-9]+)'
+          PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)'
           mkdir -p repo/org/apache/arrow/
-          for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.pom' | grep dev); do
+          for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.pom' | grep SNAPSHOT); do
             [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]]
             mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
+            mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
+            # Copy twice to maintain a latest snapshot and some earlier versions
             cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]}
+            cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE}
             echo "Artifacts $LIBRARY configured"
           done
       - name: Prune Repository

.github/workflows/python.yml

@@ -160,7 +160,10 @@ jobs:
             -r python/requirements-test.txt
       - name: Build
         shell: bash
+        # use brew version of clang, to be consistent with LLVM lib, see ARROW-17790.
         run: |
+          export CC=$(brew --prefix llvm)/bin/clang
+          export CXX=$(brew --prefix llvm)/bin/clang++
           export PYTHON=python3
           ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
           ci/scripts/python_build.sh $(pwd) $(pwd)/build

.github/workflows/r.yml

@@ -119,12 +119,13 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - { org: "rstudio", image: "r-base", tag: "4.0-centos7" }
-          - { org: "rhub", image: "debian-gcc-devel", tag: "latest" }
+          - { org: "rstudio", image: "r-base", tag: "4.0-centos7", devtoolset: "8" }
+          - { org: "rhub", image: "debian-gcc-devel", tag: "latest", devtoolset: "" }
     env:
       R_ORG: ${{ matrix.config.org }}
       R_IMAGE: ${{ matrix.config.image }}
       R_TAG: ${{ matrix.config.tag }}
+      DEVTOOLSET_VERSION: ${{ matrix.config.devtoolset }}
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v3
@@ -170,8 +171,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - { rtools: 35, arch: 'mingw32' }
-        - { rtools: 35, arch: 'mingw64' }
         - { rtools: 40, arch: 'mingw32' }
         - { rtools: 40, arch: 'mingw64' }
         - { rtools: 40, arch: 'ucrt64' }
@@ -186,11 +185,6 @@ jobs:
         run: |
           ci/scripts/ccache_setup.sh
           echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV
-      # We must enable actions/cache before r-lib/actions/setup-r to ensure
-      # using system tar instead of tar provided by Rtools.
-      # We can use tar provided by Rtools when we drop support for Rtools 3.5.
-      # Because Rtools 4.0 or later has zstd. actions/cache requires zstd
-      # when tar is GNU tar.
       - name: Cache ccache
         uses: actions/cache@v2
         with:
@@ -199,19 +193,11 @@ jobs:
           restore-keys: |
             r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-${{ hashFiles('cpp/src/**/*.cc','cpp/src/**/*.h)') }}-
             r-${{ matrix.config.rtools }}-ccache-mingw-${{ matrix.config.arch }}-
-      # We use the makepkg-mingw setup that is included in rtools40 even when
-      # we use the rtools35 compilers, so we always install R 4.0/Rtools40
       - uses: r-lib/actions/setup-r@v2
         with:
           r-version: "4.1"
           rtools-version: 40
           Ncpus: 2
-      - uses: r-lib/actions/setup-r@v2
-        if: ${{ matrix.config.rtools == 35 }}
-        with:
-          rtools-version: 35
-          r-version: "3.6"
-          Ncpus: 2
       - name: Build Arrow C++
         shell: bash
         env:
@@ -226,11 +212,6 @@ jobs:
         with:
           name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
           path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
-      # We can remove this when we drop support for Rtools 3.5.
-      - name: Ensure using system tar in actions/cache
-        run: |
-          Write-Output "${Env:windir}\System32" | `
-            Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
   windows-r:
     needs: [windows-cpp]
@@ -242,7 +223,6 @@ jobs:
       fail-fast: false
       matrix:
         config:
-        - { rtools: 35, rversion: "3.6" }
         - { rtools: 40, rversion: "4.1" }
         - { rtools: 42, rversion: "4.2" }
         - { rtools: 42, rversion: "devel" }
@@ -284,8 +264,6 @@ jobs:
         with:
           r-version: ${{ matrix.config.rversion }}
           rtools-version: ${{ matrix.config.rtools }}
-          # RSPM keeps install times short for 3.6
-          use-public-rspm: true
           Ncpus: 2
       - uses: r-lib/actions/setup-r-dependencies@v2
         env:

.github/workflows/ruby.yml

@@ -164,7 +164,10 @@ jobs:
           key: ruby-ccache-macos-${{ hashFiles('cpp/**') }}
           restore-keys: ruby-ccache-macos-
       - name: Build C++
+        # use brew version of clang, to be consistent with LLVM lib, see ARROW-17790.
         run: |
+          export CC=$(brew --prefix llvm)/bin/clang
+          export CXX=$(brew --prefix llvm)/bin/clang++
           ci/scripts/cpp_build.sh $(pwd) $(pwd)/build
       - name: Build GLib
         run: |
@@ -195,7 +198,8 @@ jobs:
       ARROW_BUILD_TYPE: release
       ARROW_FLIGHT: ON
       ARROW_FLIGHT_SQL: ON
-      ARROW_GANDIVA: ON
+      # ARROW-17728: SEGV on MinGW
+      ARROW_GANDIVA: OFF
       ARROW_GCS: ON
       ARROW_HDFS: OFF
       ARROW_HOME: /ucrt${{ matrix.mingw-n-bits }}

.travis.yml

@@ -161,6 +161,8 @@ jobs:
 
   allow_failures:
     - name: "Java on s390x"
+    - name: "C++ on s390x"
+    - name: "Python on s390x"
 
 before_install:
   - eval "$(python ci/detect-changes.py)"