ARROW-14171: [C++][Python][Packaging] Upgrade VCPKG version and add google-cloud-cpp dependency

.env

@@ -36,7 +36,7 @@ BUILDKIT_INLINE_CACHE=1
 # different architecture notations
 ARCH=amd64
 ARCH_ALIAS=x86_64
-ARCH_SHORT_ALIAS=x64
+ARCH_SHORT=amd64
 
 ULIMIT_CORE=-1
 REPO=apache/arrow-dev
@@ -70,6 +70,11 @@ TZ=UTC
 # -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
 DEVTOOLSET_VERSION=-1
 
-# Used for the manylinux and windows wheels, please update the crossbow configuration on update:
+# Used through docker-compose.yml and serves as the default version for the
+# ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
+# docker tags more readable.
+#
+# Please also update the crossbow configuration in order to keep the github
+# actions cache up to date for the macOS wheels:
 #   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-VCPKG="2021.04.30"
+VCPKG="30465138e"

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -18,65 +18,61 @@
 ARG base
 FROM ${base}
 
-ARG arch_alias
-ARG arch_short_alias
+ARG arch
+ARG arch_short
+ARG manylinux
 
+ENV MANYLINUX_VERSION=${manylinux}
+
+# Install basic dependencies
 RUN yum install -y git flex curl autoconf zip wget
 
 # Install CMake
-ARG cmake=3.19.3
-RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \
-    tar -xzf - --directory /usr/local --strip-components=1
+# AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
+ARG cmake=3.21.4
+COPY ci/scripts/install_cmake.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local
 
 # Install Ninja
 ARG ninja=1.10.2
-RUN mkdir /tmp/ninja && \
-    wget -q https://github.com/ninja-build/ninja/archive/v${ninja}.tar.gz -O - | \
-    tar -xzf - --directory /tmp/ninja --strip-components=1 && \
-    cd /tmp/ninja && \
-    ./configure.py --bootstrap && \
-    mv ninja /usr/local/bin && \
-    rm -rf /tmp/ninja
+COPY ci/scripts/install_ninja.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ninja.sh ${ninja} /usr/local
 
 # Install ccache
 ARG ccache=4.1
-RUN mkdir /tmp/ccache && \
-    wget -q https://github.com/ccache/ccache/archive/v${ccache}.tar.gz -O - | \
-    tar -xzf - --directory /tmp/ccache --strip-components=1 && \
-    cd /tmp/ccache && \
-    mkdir build && \
-    cd build && \
-    cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON .. && \
-    ninja install && \
-    rm -rf /tmp/ccache
+COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
 
-# Install vcpkg
+# Install vcpkg and in case of manylinux2010 install a more recent glibc>2.15
+# for the prebuilt vcpkg binary
 ARG vcpkg
-RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \
-    git -C /opt/vcpkg checkout ${vcpkg} && \
-    /opt/vcpkg/bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics && \
-    ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg
-
-# Patch ports files as needed
+ARG glibc=2.18
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
      arrow/ci/vcpkg/
-RUN cd /opt/vcpkg && git apply --ignore-whitespace /arrow/ci/vcpkg/ports.patch
+COPY ci/scripts/install_vcpkg.sh \
+     ci/scripts/install_glibc.sh \
+     arrow/ci/scripts/
+RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+    if [ "${manylinux}" == "2010" ]; then \
+        arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
+        patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
+        patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+    fi
+ENV PATH="/opt/vcpkg:${PATH}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_FORCE_SYSTEM_BINARIES=1 \
     VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
-    VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \
+    VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
     VCPKG_FEATURE_FLAGS=-manifests
 
 # Need to install the boost-build prior installing the boost packages, otherwise
 # vcpkg will raise an error.
 # TODO(kszucs): factor out the package enumeration to a text file and reuse it
 # from the windows image and potentially in a future macos wheel build
 RUN vcpkg install --clean-after-build \
-        boost-build:${arch_short_alias}-linux && \
-    vcpkg install --clean-after-build \
         abseil \
         aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
         boost-filesystem \
@@ -87,6 +83,7 @@ RUN vcpkg install --clean-after-build \
         flatbuffers \
         gflags \
         glog \
+        google-cloud-cpp[core,storage] \
         grpc \
         lz4 \
         openssl \

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -17,7 +17,7 @@
 
 # based on mcr.microsoft.com/windows/servercore:ltsc2019
 # contains choco and vs2017 preinstalled
-FROM abrarov/msvc-2017:2.10.0
+FROM abrarov/msvc-2017:2.12.1
 
 # Install CMake and Ninja
 RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' && \
@@ -31,16 +31,12 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin"
 # Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has
 # started to ship precompiled binaries for the vcpkg-tool.
 ARG vcpkg
-RUN git clone https://github.com/Microsoft/vcpkg && \
-    vcpkg\bootstrap-vcpkg.bat -disableMetrics && \
-    setx PATH "%PATH%;C:\vcpkg" && \
-    git -C vcpkg checkout %vcpkg%
-
-# Patch ports files as needed
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*windows*.cmake \
      arrow/ci/vcpkg/
-RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch
+COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
+    setx PATH "%PATH%;C:\vcpkg"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
@@ -50,7 +46,7 @@ RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
-    VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \
+    VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
     VCPKG_FEATURE_FLAGS=-manifests
 
 RUN vcpkg install --clean-after-build \
@@ -66,6 +62,7 @@ RUN vcpkg install --clean-after-build \
         flatbuffers \
         gflags \
         glog \
+        google-cloud-cpp[core,storage] \
         grpc \
         lz4 \
         openssl \

ci/scripts/install_ccache.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <prefix>"
+  exit 1
+fi
+
+version=$1
+prefix=$2
+url="https://github.com/ccache/ccache/archive/v${version}.tar.gz"
+
+mkdir /tmp/ccache
+wget -q ${url} -O - | tar -xzf - --directory /tmp/ccache --strip-components=1
+
+mkdir /tmp/ccache/build
+pushd /tmp/ccache/build
+cmake \
+  -GNinja \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX=${prefix} \
+  -DZSTD_FROM_INTERNET=ON \
+  ..
+ninja install
+popd
+
+rm -rf /tmp/ccache

ci/scripts/install_cmake.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+declare -A archs
+archs=([amd64]=x86_64
+       [arm64v8]=aarch64)
+
+declare -A platforms
+platforms=([linux]=linux
+           [macos]=macos
+           [windows]=windows)
+
+if [ "$#" -ne 4 ]; then
+  echo "Usage: $0 <architecture> <platform> <version> <prefix>"
+  exit 1
+fi
+
+arch=${archs[$1]}
+platform=${platforms[$2]}
+version=$3
+prefix=$4
+
+url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-${arch}.tar.gz"
+wget -q ${url} -O - | tar -xzf - --directory ${prefix} --strip-components=1

ci/scripts/install_glibc.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <prefix>"
+  exit 1
+fi
+
+version=$1
+prefix=$2
+
+url="http://ftp.gnu.org/gnu/glibc/glibc-${version}.tar.gz"
+
+mkdir /tmp/glibc
+wget -q ${url} -O - | tar -xzf - --directory /tmp/glibc --strip-components=1
+
+mkdir /tmp/glibc/build
+pushd /tmp/glibc/build
+../configure --prefix=${prefix}
+make -j$(nproc)
+make install
+popd
+
+rm -rf /tmp/glibc

ci/scripts/install_ninja.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <prefix>"
+  exit 1
+fi
+
+version=$1
+prefix=$2
+
+url="https://github.com/ninja-build/ninja/archive/v${version}.tar.gz"
+
+mkdir /tmp/ninja
+wget -q ${url} -O - | tar -xzf - --directory /tmp/ninja --strip-components=1
+
+pushd /tmp/ninja
+./configure.py --bootstrap
+mv ninja ${prefix}/bin
+popd
+
+rm -rf /tmp/ninja

ci/scripts/install_vcpkg.sh

@@ -19,21 +19,35 @@
 
 set -e
 
-if [ "$#" -ne 2 ]; then
-  echo "Usage: $0 <vcpkg version> <target directory>"
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 ``<target-directory> [<vcpkg-version> [<vcpkg-ports-patch>]]"
   exit 1
 fi
 
-vcpkg_version=$1
-vcpkg_destination=$2
-vcpkg_patch=$(realpath $(dirname "${0}")/../vcpkg/ports.patch)
+arrow_dir=$(cd -- "$(dirname -- "$0")/../.." && pwd -P)
+default_vcpkg_version=$(cat "${arrow_dir}/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
+default_vcpkg_ports_patch="${arrow_dir}/ci/vcpkg/ports.patch"
 
-git clone --depth 1 --branch ${vcpkg_version} https://github.com/microsoft/vcpkg ${vcpkg_destination}
+vcpkg_destination=$1
+vcpkg_version=${2:-$default_vcpkg_version}
+vcpkg_ports_patch=${3:-$default_vcpkg_ports_patch}
+
+# reduce the fetched data using a shallow clone
+git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}
 
 pushd ${vcpkg_destination}
 
-./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics
-git apply --ignore-whitespace ${vcpkg_patch}
-echo "Patch successfully applied!"
+git checkout "${vcpkg_version}"
+
+if [[ "$OSTYPE" == "msys" ]]; then
+  ./bootstrap-vcpkg.bat -disableMetrics
+else
+  ./bootstrap-vcpkg.sh -disableMetrics
+fi
+
+if [ -f "${vcpkg_ports_patch}" ]; then
+  git apply --verbose --ignore-whitespace ${vcpkg_ports_patch}
+  echo "Patch successfully applied to the VCPKG port files!"
+fi
 
 popd