Update Arrow to apache-arrow-4.0.1

.env

@@ -18,31 +18,57 @@
 # All of the following environment variables are required to set default values
 # for the parameters in docker-compose.yml.
 
+# empty prefix means that the docker-compose configuration will use named
+# volumes which potentially improves the performance on docker for macos and
+# docker for windows, it also prevents the contamination of the source
+# directory
+# a non-empty prefix means that directories from the host are bind-mounted
+# into the container, it should be set to ".docker/" on github actions to keep
+# the cache plugin functional
+DOCKER_VOLUME_PREFIX=
+
+# turn on inline build cache, this is a docker buildx feature documented
+# at https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
+COMPOSE_DOCKER_CLI_BUILD=1
+DOCKER_BUILDKIT=1
+BUILDKIT_INLINE_CACHE=1
+
+# different architecture notations
+ARCH=amd64
+ARCH_ALIAS=x86_64
+ARCH_SHORT_ALIAS=x64
+
 ULIMIT_CORE=-1
 REPO=apache/arrow-dev
-ARCH=amd64
 CUDA=9.1
 DEBIAN=10
-UBUNTU=18.04
-FEDORA=32
+UBUNTU=20.04
+FEDORA=33
 PYTHON=3.6
-LLVM=10
+LLVM=12
 CLANG_TOOLS=8
-RUST=nightly-2020-04-22
-GO=1.12
+RUST=nightly-2021-03-24
+GO=1.15
 NODE=14
 MAVEN=3.5.4
 JDK=8
+NUMPY=latest
 PANDAS=latest
 DASK=latest
 TURBODBC=latest
 KARTOTHEK=latest
-HDFS=2.9.2
+HDFS=3.2.1
 SPARK=master
-DOTNET=2.1
+DOTNET=3.1
 R=4.0
 ARROW_R_DEV=TRUE
 # These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
 R_ORG=rhub
 R_IMAGE=ubuntu-gcc-release
 R_TAG=latest
+# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
+DEVTOOLSET_VERSION=-1
+
+# Used for the manylinux and windows wheels, please update the crossbow configuration on update:
+#   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
+VCPKG=fced4bef1606260f110d74de1ae1975c2b9ac549

.github/workflows/archery.yml

@@ -43,23 +43,22 @@ jobs:
         with:
           fetch-depth: 0
       - name: Git Fixup
-        if: ${{ github.event_name == 'pull_request' }}
+        if: ${{ github.ref != 'master' }}
         shell: bash
         run: git branch master origin/master
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Setup Python
         uses: actions/setup-python@v1
         with:
-          python-version: '3.5'
+          python-version: '3.6'
       - name: Install Archery, Crossbow- and Test Dependencies
-        working-directory: dev/archery
-        run: pip install pytest responses toolz jinja2 -e .[all]
+        run: pip install pytest responses -e dev/archery[all]
       - name: Archery Unittests
         working-directory: dev/archery
         run: pytest -v archery
       - name: Archery Docker Validation
         run: archery docker
       - name: Crossbow Check Config
         working-directory: dev/tasks
-        run: python crossbow.py check-config
+        run: archery crossbow check-config

.github/workflows/cancel.yml

@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Cancel stale runs
+
+on:
+  workflow_run:
+    # The name of another workflow (whichever one) that always runs on PRs
+    workflows: ['Dev']
+    types: ['requested']
+
+jobs:
+  cancel-stale-workflow-runs:
+    name: "Cancel stale workflow runs"
+    runs-on: ubuntu-latest
+    steps:
+      # Unfortunately, we need to define a separate cancellation step for
+      # each workflow where we want to cancel stale runs.
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale C++ runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: cpp.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale C# runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: csharp.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Dev runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: dev.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Go runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: go.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Integration runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: integration.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Java JNI runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: java_jni.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Java runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: java.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale JS runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: js.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Julia runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: julia.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Python runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: python.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale R runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: r.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Ruby runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: ruby.yml
+          skipEventTypes: '["push", "schedule"]'
+      - uses: potiuk/cancel-workflow-runs@master
+        name: "Cancel stale Rust runs"
+        with:
+          cancelMode: allDuplicates
+          token: ${{ secrets.GITHUB_TOKEN }}
+          workflowFileName: rust.yml
+          skipEventTypes: '["push", "schedule"]'

.github/workflows/comment_bot.yml

@@ -34,17 +34,12 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: arrow
-      # because libgit2 is a dependency of crossbow so prefer conda
-      - name: Setup Conda
-        uses: s-weigand/setup-conda@v1
+      - name: Set up Python
+        uses: actions/setup-python@v2
         with:
-          update-conda: true
-          python-version: 3.7
-          conda-channels: conda-forge
-      - name: Instal Archery and Crossbow dependencies
-        run: |
-          conda install -y --file arrow/ci/conda_env_crossbow.txt pygithub
-          pip install -e arrow/dev/archery[bot]
+          python-version: 3.8
+      - name: Install Archery and Crossbow dependencies
+        run: pip install -e arrow/dev/archery[bot]
       - name: Handle Github comment event
         env:
           ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -72,16 +67,16 @@ jobs:
             git diff --name-only HEAD..upstream/master | grep -e "$1" >/dev/null 2>&1
           }
           if changed '^r/.*\.R$'; then
-            echo "::set-env name=R_DOCS::true"
+            echo "R_DOCS=true" >> $GITHUB_ENV
           fi
           if changed 'cmake' || changed 'CMake'; then
-            echo "::set-env name=CMAKE_FORMAT::true"
+            echo "CMAKE_FORMAT=true" >> $GITHUB_ENV
           fi
           if changed '^cpp/src'; then
-            echo "::set-env name=CLANG_FORMAT_CPP::true"
+            echo "CLANG_FORMAT_CPP=true" >> $GITHUB_ENV
           fi
           if changed '^r/src'; then
-            echo "::set-env name=CLANG_FORMAT_R::true"
+            echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV
           fi
       - name: Run cmake_format
         if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything')

.github/workflows/cpp.yml

@@ -38,9 +38,8 @@ on:
       - 'format/Flight.proto'
 
 env:
-  DOCKER_BUILDKIT: 0
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
+  DOCKER_VOLUME_PREFIX: ".docker/"
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -60,7 +59,7 @@ jobs:
           - image: conda-cpp
             title: AMD64 Conda C++
           - image: ubuntu-cpp-sanitizer
-            title: AMD64 Ubuntu 18.04 C++ ASAN UBSAN
+            title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN
     steps:
       - name: Checkout Arrow
         uses: actions/checkout@v2
@@ -71,7 +70,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
@@ -218,7 +217,9 @@ jobs:
       - name: Install Dependencies
         shell: bash
         run: |
+          rm -f /usr/local/bin/2to3
           brew update --preinstall
+          brew unlink gcc@8 gcc@9
           brew bundle --file=cpp/Brewfile
       - name: Build
         shell: bash
@@ -338,7 +339,6 @@ jobs:
       CMAKE_ARGS: >-
         -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }}
         -DBoost_NO_BOOST_CMAKE=ON
-      CMAKE_GENERATOR: MSYS Makefiles
       CMAKE_UNITY_BUILD: ON
     steps:
       - name: Disable Crash Dialogs
@@ -365,7 +365,7 @@ jobs:
         run: |
           ci/scripts/msys2_setup.sh cpp
       - name: Cache ccache
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ccache
           key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }}

.github/workflows/cpp_cron.yml

@@ -30,9 +30,8 @@ on:
         0 */12 * * *
 
 env:
-  DOCKER_BUILDKIT: 0
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
+  DOCKER_VOLUME_PREFIX: ".docker/"
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -46,36 +45,31 @@ jobs:
       matrix:
         name:
           - amd64-debian-10-cpp
-          - amd64-fedora-32-cpp
-          - amd64-ubuntu-16.04-cpp
+          - amd64-fedora-33-cpp
           - amd64-ubuntu-18.04-cpp
-          - amd64-ubuntu-18.04-cpp-cmake32
+          - amd64-ubuntu-20.04-cpp
         include:
           - name: amd64-debian-10-cpp
             image: debian-cpp
             title: AMD64 Debian 10 C++
             debian: 10
-          - name: amd64-fedora-32-cpp
+          - name: amd64-fedora-33-cpp
             image: fedora-cpp
-            title: AMD64 Fedora 32 C++
-            fedora: 32
-          - name: amd64-ubuntu-16.04-cpp
-            image: ubuntu-cpp
-            title: AMD64 Ubuntu 16.04 C++
-            ubuntu: 16.04
+            title: AMD64 Fedora 33 C++
+            fedora: 33
           - name: amd64-ubuntu-18.04-cpp
             image: ubuntu-cpp
             title: AMD64 Ubuntu 18.04 C++
             ubuntu: 18.04
-          - name: amd64-ubuntu-18.04-cpp-cmake32
-            image: ubuntu-cpp-cmake32
-            title: AMD64 Ubuntu 18.04 C++ CMake 3.2
-            ubuntu: 18.04
+          - name: amd64-ubuntu-20.04-cpp
+            image: ubuntu-cpp
+            title: AMD64 Ubuntu 20.04 C++
+            ubuntu: 20.04
     env:
       # the defaults here should correspond to the values in .env
       ARCH: 'amd64'
       DEBIAN: ${{ matrix.debian || 10 }}
-      FEDORA: ${{ matrix.fedora || 32 }}
+      FEDORA: ${{ matrix.fedora || 33 }}
       UBUNTU: ${{ matrix.ubuntu || 18.04 }}
     steps:
       - name: Checkout Arrow
@@ -87,7 +81,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ${{ matrix.name }}-${{ hashFiles('cpp/**') }}