From b7bb816e6326bf49dd1e89966279ee5a9fcf7b79 Mon Sep 17 00:00:00 2001
From: Carlos O'Ryan <coryan@google.com>
Date: Thu, 28 Oct 2021 13:00:21 +0000
Subject: [PATCH 01/33] ARROW-14171: [C++] add google-cloud-cpp to vcpkg

---
 ci/docker/python-wheel-manylinux-201x.dockerfile | 1 +
 ci/docker/python-wheel-windows-vs2017.dockerfile | 1 +
 cpp/vcpkg.json                                   | 9 +++++++++
 dev/tasks/python-wheels/github.osx.amd64.yml     | 5 +++++
 dev/tasks/python-wheels/github.osx.arm64.yml     | 1 +
 5 files changed, 17 insertions(+)

diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
index ae1b0a7767c..e62265e3951 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -87,6 +87,7 @@ RUN vcpkg install --clean-after-build \
         flatbuffers \
         gflags \
         glog \
+        google-cloud-cpp[core,storage] \
         grpc \
         lz4 \
         openssl \
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
index 9a2afb781fa..fc80677d03a 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -66,6 +66,7 @@ RUN vcpkg install --clean-after-build \
         flatbuffers \
         gflags \
         glog \
+        google-cloud-cpp[core,storage] \
         grpc \
         lz4 \
         openssl \
diff --git a/cpp/vcpkg.json b/cpp/vcpkg.json
index f74b4d3e37b..a6e9c68b57d 100644
--- a/cpp/vcpkg.json
+++ b/cpp/vcpkg.json
@@ -17,6 +17,7 @@
     "benchmark",
     "boost-filesystem",
     "boost-multiprecision",
+    "boost-process",
     "boost-system",
     "brotli",
     "bzip2",
@@ -25,6 +26,14 @@
     "flatbuffers",
     "gflags",
     "glog",
+    {
+      "name":"google-cloud-cpp",
+      "version>=": "1.32.1",
+      "default-features": false,
+      "features": [
+        "storage"
+      ]
+    },
     "grpc",
     "gtest",
     "lz4",
diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index 8078abfd56f..42458e790a1 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -86,6 +86,11 @@ jobs:
         run: $VCPKG_ROOT/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer]
       {% endif %}
 
+      {% if arrow_gcs == "ON" %}
+      - name: Install Google Cloud Platform C++
+        run: $VCPKG_ROOT/vcpkg install google-cloud-cpp[core,storage]
+      {% endif %}
+
       - name: Install Python {{ python_version }}
         shell: bash
         run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }}
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index e5456dbfc28..2d10626590b 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -97,6 +97,7 @@ jobs:
             flatbuffers \
             gflags \
             glog \
+            google-cloud-cpp[core,storage] \
             lz4 \
             orc \
             protobuf \

From 8b6755d549d0b4e5c0aac8fda153b56ea865ee8a Mon Sep 17 00:00:00 2001
From: Carlos O'Ryan <coryan@google.com>
Date: Thu, 28 Oct 2021 17:04:58 +0000
Subject: [PATCH 02/33] Revent changes to osx Jinja template

I probably do not understand how these are used, probably something for
later.
---
 dev/tasks/python-wheels/github.osx.amd64.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index 42458e790a1..8078abfd56f 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -86,11 +86,6 @@ jobs:
         run: $VCPKG_ROOT/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer]
       {% endif %}
 
-      {% if arrow_gcs == "ON" %}
-      - name: Install Google Cloud Platform C++
-        run: $VCPKG_ROOT/vcpkg install google-cloud-cpp[core,storage]
-      {% endif %}
-
       - name: Install Python {{ python_version }}
         shell: bash
         run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }}

From 0dfbc391eafe601b9d29880b2b718c6044f64b24 Mon Sep 17 00:00:00 2001
From: Carlos O'Ryan <coryan@google.com>
Date: Thu, 4 Nov 2021 12:56:53 +0000
Subject: [PATCH 03/33] Address review comments

---
 .env                | 2 +-
 dev/tasks/tasks.yml | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.env b/.env
index 43cfefccee4..48850606916 100644
--- a/.env
+++ b/.env
@@ -72,4 +72,4 @@ DEVTOOLSET_VERSION=-1
 
 # Used for the manylinux and windows wheels, please update the crossbow configuration on update:
 #   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-VCPKG="2021.04.30"
+VCPKG="9675ab15def74a0b96d954bddc7ddabb42c92452"
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 70926ef8af5..0c346817ab0 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -399,7 +399,7 @@ tasks:
     params:
       arch: arm64
       arrow_simd_level: "DEFAULT"
-      vcpkg_version: "2021.04.30"
+      vcpkg_version: "9675ab15def74a0b96d954bddc7ddabb42c92452"
       python_version: "3.8"
       macos_deployment_target: "11.0"
 
@@ -413,7 +413,7 @@ tasks:
     params:
       arch: arm64
       arrow_simd_level: "DEFAULT"
-      vcpkg_version: "2021.04.30"
+      vcpkg_version: "9675ab15def74a0b96d954bddc7ddabb42c92452"
       python_version: "{{ python_version }}"
       macos_deployment_target: "11.0"
     artifacts:
@@ -427,7 +427,7 @@ tasks:
       # Universal2 builds for both rosetta and native, but we currently can't
       # configure SIMD for both architectures at the same time
       arrow_simd_level: "NONE"
-      vcpkg_version: "2021.04.30"
+      vcpkg_version: "9675ab15def74a0b96d954bddc7ddabb42c92452"
       python_version: "{{ python_version }}"
       macos_deployment_target: "10.13"
     artifacts:

From a5beaced329d9268b533a652580920d2720597e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Tue, 23 Nov 2021 14:28:29 +0100
Subject: [PATCH 04/33] Update vcpkg patch; support installing specific vcpkg
 SHA

---
 ci/scripts/install_vcpkg.sh |  3 ++-
 ci/vcpkg/ports.patch        | 35 +++++++++++------------------------
 2 files changed, 13 insertions(+), 25 deletions(-)

diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index fe99a7fea2f..cb11667402d 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -28,10 +28,11 @@ vcpkg_version=$1
 vcpkg_destination=$2
 vcpkg_patch=$(realpath $(dirname "${0}")/../vcpkg/ports.patch)
 
-git clone --depth 1 --branch ${vcpkg_version} https://github.com/microsoft/vcpkg ${vcpkg_destination}
+git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}
 
 pushd ${vcpkg_destination}
 
+git checkout ${vcpkg_version}
 ./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics
 git apply --ignore-whitespace ${vcpkg_patch}
 echo "Patch successfully applied!"
diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index 7bcba49c194..cdbfb716c4c 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -1,46 +1,33 @@
-diff --git a/ports/aws-c-common/portfile.cmake b/ports/aws-c-common/portfile.cmake
-index f3704ef05b..3af543058d 100644
---- a/ports/aws-c-common/portfile.cmake
-+++ b/ports/aws-c-common/portfile.cmake
-@@ -1,8 +1,8 @@
- vcpkg_from_github(
-     OUT_SOURCE_PATH SOURCE_PATH
-     REPO awslabs/aws-c-common
--    REF 4a21a1c0757083a16497fea27886f5f20ccdf334 # v0.4.56
--    SHA512 68898a8ac15d5490f45676eabfbe0df9e45370a74c543a28909fd0d85fed48dfcf4bcd6ea2d01d1a036dd352e2e4e0b08c48c63ab2a2b477fe150b46a827136e
-+    REF 13adef72b7813ec878817c6d50a7a3f241015d8a # v0.4.57
-+    SHA512 28256522ac6af544d7464e3e7dcd4dc802ae2b09728bf8f167f86a6487bb756d0cad5eb4a2480610b2967b9c24c4a7f70621894517aa2828ffdeb0479453803b
-     HEAD_REF master
-     PATCHES
-         disable-error-4068.patch # This patch fixes dependency port compilation failure
 diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
-index be66d452be..a5ce325e9d 100644
+index 02977602..edb953c2 100644
 --- a/ports/curl/portfile.cmake
 +++ b/ports/curl/portfile.cmake
-@@ -94,6 +94,8 @@ vcpkg_configure_cmake(
+@@ -91,6 +91,10 @@ vcpkg_cmake_configure(
          -DCMAKE_DISABLE_FIND_PACKAGE_Perl=ON
          -DENABLE_DEBUG=ON
          -DCURL_CA_FALLBACK=ON
 +        -DCURL_CA_PATH=none
 +        -DCURL_CA_BUNDLE=none
-     OPTIONS_DEBUG
-         ${EXTRA_ARGS_DEBUG}
++    OPTIONS_DEBUG
++        ${EXTRA_ARGS_DEBUG}
      OPTIONS_RELEASE
+         ${OPTIONS_RELEASE}
+     OPTIONS_DEBUG
 diff --git a/ports/snappy/portfile.cmake b/ports/snappy/portfile.cmake
-index 75dd133027..84345c7caa 100644
+index 07bef5ee..d58089bc 100644
 --- a/ports/snappy/portfile.cmake
 +++ b/ports/snappy/portfile.cmake
 @@ -4,6 +4,7 @@ vcpkg_from_github(
-     REF 537f4ad6240e586970fe554614542e9717df7902 # 1.1.8
-     SHA512 555d3b69a6759592736cbaae8f41654f0cf14e8be693b5dde37640191e53daec189f895872557b173e905d10681ef502f3e6ed8566811add963ffef96ce4016d
+     REF 1.1.9
+     SHA512 F1F8A90F5F7F23310423574B1D8C9ACB84C66EA620F3999D1060395205E5760883476837ABA02F0AA913AF60819E34C625D8308C18A5D7A9C4E190F35968B024
      HEAD_REF master
 +    PATCHES "snappy-disable-bmi.patch"
  )
-
+ 
  vcpkg_configure_cmake(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
-index 0000000000..2cbb1533a8
+index 00000000..2cbb1533
 --- /dev/null
 +++ b/ports/snappy/snappy-disable-bmi.patch
 @@ -0,0 +1,17 @@

From 8e3acb20912e8706aee7e1c7ebd7348b92bf5f74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Tue, 23 Nov 2021 14:43:43 +0100
Subject: [PATCH 05/33] Pin vcpkg HEAD

---
 .env                                             | 2 +-
 ci/docker/python-wheel-windows-vs2017.dockerfile | 2 +-
 ci/scripts/install_vcpkg.sh                      | 2 +-
 ci/vcpkg/ports.patch                             | 6 ++++--
 4 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.env b/.env
index 48850606916..a2bc20214c2 100644
--- a/.env
+++ b/.env
@@ -72,4 +72,4 @@ DEVTOOLSET_VERSION=-1
 
 # Used for the manylinux and windows wheels, please update the crossbow configuration on update:
 #   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-VCPKG="9675ab15def74a0b96d954bddc7ddabb42c92452"
+VCPKG="30465138ef7facf1d4c1aa8a06dd62ceb71cc2eb"
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
index fc80677d03a..1bd3dbca4de 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -40,7 +40,7 @@ RUN git clone https://github.com/Microsoft/vcpkg && \
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*windows*.cmake \
      arrow/ci/vcpkg/
-RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch
+RUN cd vcpkg && git apply --verbose --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index cb11667402d..00537e92c7a 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -34,7 +34,7 @@ pushd ${vcpkg_destination}
 
 git checkout ${vcpkg_version}
 ./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics
-git apply --ignore-whitespace ${vcpkg_patch}
+git apply --verbose --ignore-whitespace ${vcpkg_patch}
 echo "Patch successfully applied!"
 
 popd
diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index cdbfb716c4c..77b8372525e 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -18,12 +18,14 @@ index 07bef5ee..d58089bc 100644
 --- a/ports/snappy/portfile.cmake
 +++ b/ports/snappy/portfile.cmake
 @@ -4,6 +4,7 @@ vcpkg_from_github(
+     OUT_SOURCE_PATH SOURCE_PATH
+     REPO google/snappy
      REF 1.1.9
-     SHA512 F1F8A90F5F7F23310423574B1D8C9ACB84C66EA620F3999D1060395205E5760883476837ABA02F0AA913AF60819E34C625D8308C18A5D7A9C4E190F35968B024
+     SHA512 f1f8a90f5f7f23310423574b1d8c9acb84c66ea620f3999d1060395205e5760883476837aba02f0aa913af60819e34c625d8308c18a5d7a9c4e190f35968b024
      HEAD_REF master
 +    PATCHES "snappy-disable-bmi.patch"
  )
- 
+
  vcpkg_configure_cmake(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644

From 27dfacc41a3340dc576eb33136cdd2418810eb58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 24 Nov 2021 01:30:05 +0100
Subject: [PATCH 06/33] Refactor VCPKG installation

---
 .env                                          |  9 +++--
 .../python-wheel-manylinux-201x.dockerfile    | 12 +++----
 .../python-wheel-windows-vs2017.dockerfile    | 10 ++----
 ci/scripts/install_vcpkg.sh                   | 23 +++++++++----
 ci/vcpkg/ports.patch                          | 34 +++++++++----------
 dev/tasks/python-wheels/github.osx.amd64.yml  |  5 ++-
 dev/tasks/python-wheels/github.osx.arm64.yml  |  3 +-
 dev/tasks/tasks.yml                           |  4 ---
 docker-compose.yml                            |  2 +-
 9 files changed, 51 insertions(+), 51 deletions(-)

diff --git a/.env b/.env
index a2bc20214c2..088742ed570 100644
--- a/.env
+++ b/.env
@@ -70,6 +70,11 @@ TZ=UTC
 # -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
 DEVTOOLSET_VERSION=-1
 
-# Used for the manylinux and windows wheels, please update the crossbow configuration on update:
+# Used through docker-compose.yml and serves as the default version for the
+# ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
+# docker tags more readable.
+#
+# Please also update the crossbow configuration in order to keep the github
+# actions cache up to date for the macOS wheels:
 #   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-VCPKG="30465138ef7facf1d4c1aa8a06dd62ceb71cc2eb"
+VCPKG="30465138e"
diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
index e62265e3951..dbd899dafe6 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -24,7 +24,7 @@ ARG arch_short_alias
 RUN yum install -y git flex curl autoconf zip wget
 
 # Install CMake
-ARG cmake=3.19.3
+ARG cmake=3.22.0
 RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \
     tar -xzf - --directory /usr/local --strip-components=1
 
@@ -52,16 +52,12 @@ RUN mkdir /tmp/ccache && \
 
 # Install vcpkg
 ARG vcpkg
-RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \
-    git -C /opt/vcpkg checkout ${vcpkg} && \
-    /opt/vcpkg/bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics && \
-    ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg
-
-# Patch ports files as needed
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
      arrow/ci/vcpkg/
-RUN cd /opt/vcpkg && git apply --ignore-whitespace /arrow/ci/vcpkg/ports.patch
+COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg}
+ENV PATH="/opt/vcpkg:${PATH}"
 
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
index 1bd3dbca4de..234ec667918 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -31,16 +31,12 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin"
 # Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has
 # started to ship precompiled binaries for the vcpkg-tool.
 ARG vcpkg
-RUN git clone https://github.com/Microsoft/vcpkg && \
-    vcpkg\bootstrap-vcpkg.bat -disableMetrics && \
-    setx PATH "%PATH%;C:\vcpkg" && \
-    git -C vcpkg checkout %vcpkg%
-
-# Patch ports files as needed
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*windows*.cmake \
      arrow/ci/vcpkg/
-RUN cd vcpkg && git apply --verbose --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch
+COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
+    setx PATH "%PATH%;C:\vcpkg"
 
 # Configure vcpkg and install dependencies
 # NOTE: use windows batch environment notation for build arguments in RUN
diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index 00537e92c7a..73b0c568850 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -19,21 +19,30 @@
 
 set -e
 
-if [ "$#" -ne 2 ]; then
-  echo "Usage: $0 <vcpkg version> <target directory>"
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 ``<target-directory> [<vcpkg-version> [<vcpkg-ports-patch>]]"
   exit 1
 fi
 
-vcpkg_version=$1
-vcpkg_destination=$2
-vcpkg_patch=$(realpath $(dirname "${0}")/../vcpkg/ports.patch)
+arrow_dir=$(realpath $(dirname "${0}")/../..)
+default_vcpkg_patch="${arrow_dir}/ci/vcpkg/ports.patch"
+default_vcpkg_version=$(cat "${arrow_dir}/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
 
+vcpkg_destination=$1
+vcpkg_version=${2:-$default_vcpkg_version}
+vcpkg_patch=${3:-$default_vcpkg_patch}
+
+# reduce the fetched data using a shallow clone
 git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}
 
 pushd ${vcpkg_destination}
 
-git checkout ${vcpkg_version}
-./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics
+git checkout "${vcpkg_version}"
+if [[ "$OSTYPE" == "msys" ]]; then
+  ./bootstrap-vcpkg.bat -disableMetrics
+else
+  ./bootstrap-vcpkg.sh -disableMetrics
+fi
 git apply --verbose --ignore-whitespace ${vcpkg_patch}
 echo "Patch successfully applied!"
 
diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index 77b8372525e..1ce7d3d8e58 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -1,8 +1,8 @@
 diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
-index 02977602..edb953c2 100644
+index ce8b7570..8255ba24 100644
 --- a/ports/curl/portfile.cmake
 +++ b/ports/curl/portfile.cmake
-@@ -91,6 +91,10 @@ vcpkg_cmake_configure(
+@@ -92,6 +92,10 @@ vcpkg_cmake_configure(
          -DCMAKE_DISABLE_FIND_PACKAGE_Perl=ON
          -DENABLE_DEBUG=ON
          -DCURL_CA_FALLBACK=ON
@@ -14,39 +14,39 @@ index 02977602..edb953c2 100644
          ${OPTIONS_RELEASE}
      OPTIONS_DEBUG
 diff --git a/ports/snappy/portfile.cmake b/ports/snappy/portfile.cmake
-index 07bef5ee..d58089bc 100644
+index 45b8c706..b409d8a7 100644
 --- a/ports/snappy/portfile.cmake
 +++ b/ports/snappy/portfile.cmake
 @@ -4,6 +4,7 @@ vcpkg_from_github(
-     OUT_SOURCE_PATH SOURCE_PATH
-     REPO google/snappy
      REF 1.1.9
      SHA512 f1f8a90f5f7f23310423574b1d8c9acb84c66ea620f3999d1060395205e5760883476837aba02f0aa913af60819e34c625d8308c18a5d7a9c4e190f35968b024
      HEAD_REF master
 +    PATCHES "snappy-disable-bmi.patch"
  )
-
- vcpkg_configure_cmake(
+ 
+ vcpkg_cmake_configure(
 diff --git a/ports/snappy/snappy-disable-bmi.patch b/ports/snappy/snappy-disable-bmi.patch
 new file mode 100644
-index 00000000..2cbb1533
+index 00000000..a57ce0c2
 --- /dev/null
 +++ b/ports/snappy/snappy-disable-bmi.patch
-@@ -0,0 +1,17 @@
-+--- snappy.cc  2020-06-27 17:38:49.718993748 -0500
-++++ snappy.cc  2020-06-27 17:37:57.543268213 -0500
-+@@ -717,14 +717,10 @@
-+ static inline uint32 ExtractLowBytes(uint32 v, int n) {
+@@ -0,0 +1,19 @@
++diff --git a/snappy.cc b/snappy.cc
++index 79dc0e8..f3153ea 100644
++--- a/snappy.cc
+++++ b/snappy.cc
++@@ -965,14 +965,10 @@ static inline void Report(const char *algorithm, size_t compressed_size,
++ static inline uint32_t ExtractLowBytes(uint32_t v, int n) {
 +   assert(n >= 0);
 +   assert(n <= 4);
 +-#if SNAPPY_HAVE_BMI2
 +-  return _bzhi_u32(v, 8 * n);
 +-#else
-+   // This needs to be wider than uint32 otherwise `mask << 32` will be
++   // This needs to be wider than uint32_t otherwise `mask << 32` will be
 +   // undefined.
-+   uint64 mask = 0xffffffff;
++   uint64_t mask = 0xffffffff;
 +   return v & ~(mask << (8 * n));
 +-#endif
 + }
-+
-+ static inline bool LeftShiftOverflows(uint8 value, uint32 shift) {
++ 
++ static inline bool LeftShiftOverflows(uint8_t value, uint32_t shift) {
diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index 8078abfd56f..e629f0c4052 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -31,7 +31,6 @@ env:
   VCPKG_FEATURE_FLAGS: "-manifests"
   VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }}
   VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }}
-  VCPKG_VERSION: "{{ vcpkg_version }}"
 
 jobs:
   build:
@@ -47,14 +46,14 @@ jobs:
         id: vcpkg-cache
         with:
           path: vcpkg
-          key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}
+          key: vcpkg-{{ macos_deployment_target }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}
 
       - name: Install Vcpkg
         if: steps.vcpkg-cache.outputs.cache-hit != 'true'
         shell: bash
         env:
           MACOSX_DEPLOYMENT_TARGET: "10.15"
-        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT
+        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT
 
       - name: Install Packages
         run: |
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 2d10626590b..29ca3995e68 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -39,7 +39,6 @@ env:
   VCPKG_FEATURE_FLAGS: "-manifests"
   VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }}
   VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }}
-  VCPKG_VERSION: "{{ vcpkg_version }}"
 
 jobs:
   build:
@@ -60,7 +59,7 @@ jobs:
         shell: bash
         env:
           MACOSX_DEPLOYMENT_TARGET: "11.0"
-        run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT
+        run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT
 
       - name: Install OpenSSL
         shell: bash
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 0c346817ab0..9b020bb8a29 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -367,7 +367,6 @@ tasks:
     ci: github
     template: python-wheels/github.osx.amd64.yml
     params:
-      vcpkg_version: "2021.04.30"
       python_version: "{{ python_version }}"
       macos_deployment_target: {{ macos_version }}
       arrow_s3: {{ arrow_s3 }}
@@ -399,7 +398,6 @@ tasks:
     params:
       arch: arm64
       arrow_simd_level: "DEFAULT"
-      vcpkg_version: "9675ab15def74a0b96d954bddc7ddabb42c92452"
       python_version: "3.8"
       macos_deployment_target: "11.0"
 
@@ -413,7 +411,6 @@ tasks:
     params:
       arch: arm64
       arrow_simd_level: "DEFAULT"
-      vcpkg_version: "9675ab15def74a0b96d954bddc7ddabb42c92452"
       python_version: "{{ python_version }}"
       macos_deployment_target: "11.0"
     artifacts:
@@ -427,7 +424,6 @@ tasks:
       # Universal2 builds for both rosetta and native, but we currently can't
       # configure SIMD for both architectures at the same time
       arrow_simd_level: "NONE"
-      vcpkg_version: "9675ab15def74a0b96d954bddc7ddabb42c92452"
       python_version: "{{ python_version }}"
       macos_deployment_target: "10.13"
     artifacts:
diff --git a/docker-compose.yml b/docker-compose.yml
index b917edc22af..8751a8a2972 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -271,7 +271,7 @@ services:
       ARROW_USE_LD_GOLD: "ON"
       BUILD_WARNING_LEVEL: "PRODUCTION"
     volumes: *conda-volumes
-    command: 
+    command:
       ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
         /arrow/ci/scripts/cpp_test.sh /arrow /build"]
 

From 47e89352382a40d5fb06b02f01a87103e796ba31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 24 Nov 2021 12:23:03 +0100
Subject: [PATCH 07/33] Retrieve VCPKG_VERSION for github actions cache

---
 dev/tasks/python-wheels/github.osx.amd64.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index e629f0c4052..ae72c61d1db 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -42,18 +42,24 @@ jobs:
       - name: Install System Dependencies
         run: brew install bash bison coreutils ninja cmake
 
+      - name: Retrieve VCPKG version from arrow/.env
+        shell: bash
+        run: |
+          vcpkg_version=$(cat "arrow/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
+          echo "VCPKG_VERSION=$vcpkg_version" >> $GITHUB_ENV
+
       - uses: actions/cache@v2
         id: vcpkg-cache
         with:
           path: vcpkg
-          key: vcpkg-{{ macos_deployment_target }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}
+          key: vcpkg-{{ macos_deployment_target }}-{{ "${{ env.VCPKG_VERSION }}" }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}
 
       - name: Install Vcpkg
         if: steps.vcpkg-cache.outputs.cache-hit != 'true'
         shell: bash
         env:
           MACOSX_DEPLOYMENT_TARGET: "10.15"
-        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT
+        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
       - name: Install Packages
         run: |
@@ -67,6 +73,7 @@ jobs:
             flatbuffers \
             gflags \
             glog \
+            google-cloud-cpp[core,storage] \
             grpc \
             lz4 \
             openssl \

From fc00a9ce259bbeb76f9c43501e2f94a586ec6aa5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 24 Nov 2021 16:33:34 +0100
Subject: [PATCH 08/33] Pin earlier CMake due to a CMake regression

---
 ci/docker/python-wheel-manylinux-201x.dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
index dbd899dafe6..1ad7f5081ea 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -24,7 +24,8 @@ ARG arch_short_alias
 RUN yum install -y git flex curl autoconf zip wget
 
 # Install CMake
-ARG cmake=3.22.0
+# AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
+ARG cmake=3.21.4
 RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \
     tar -xzf - --directory /usr/local --strip-components=1
 

From 9a5b28ebcb5c843d04a3c1bbd4a3c579cbd82df2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Wed, 24 Nov 2021 17:17:24 +0100
Subject: [PATCH 09/33] Force build vcpkg on manylinux2010

---
 .../python-wheel-manylinux-201x.dockerfile    |  5 ++--
 ci/scripts/install_vcpkg.sh                   | 23 +++++++++++++-----
 ci/vcpkg/tool.patch                           | 24 +++++++++++++++++++
 3 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100644 ci/vcpkg/tool.patch

diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
index 1ad7f5081ea..6e1120f2e8f 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -52,12 +52,13 @@ RUN mkdir /tmp/ccache && \
     rm -rf /tmp/ccache
 
 # Install vcpkg
-ARG vcpkg
+ARG vcpkg \
+    vcpkg_download_tool="ON"
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
      arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg}
+RUN VCPKG_DOWNLOAD_TOOL=${vcpkg_download_tool} arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg}
 ENV PATH="/opt/vcpkg:${PATH}"
 
 ARG build_type=release
diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index 73b0c568850..a97af435a70 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -20,17 +20,19 @@
 set -e
 
 if [ "$#" -lt 1 ]; then
-  echo "Usage: $0 ``<target-directory> [<vcpkg-version> [<vcpkg-ports-patch>]]"
+  echo "Usage: $0 ``<target-directory> [<vcpkg-version> [[<vcpkg-ports-patch> [<vcpkg-tool-patch>]]]"
   exit 1
 fi
 
-arrow_dir=$(realpath $(dirname "${0}")/../..)
-default_vcpkg_patch="${arrow_dir}/ci/vcpkg/ports.patch"
+arrow_dir=$(readlink -f $(dirname "${0}")/../..)
 default_vcpkg_version=$(cat "${arrow_dir}/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
+default_vcpkg_tool_patch="${arrow_dir}/ci/vcpkg/tool.patch"
+default_vcpkg_ports_patch="${arrow_dir}/ci/vcpkg/ports.patch"
 
 vcpkg_destination=$1
 vcpkg_version=${2:-$default_vcpkg_version}
-vcpkg_patch=${3:-$default_vcpkg_patch}
+vcpkg_ports_patch=${3:-$default_vcpkg_ports_patch}
+vcpkg_tool_patch=${4:-$default_vcpkg_tool_patch}
 
 # reduce the fetched data using a shallow clone
 git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}
@@ -38,12 +40,21 @@ git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_
 pushd ${vcpkg_destination}
 
 git checkout "${vcpkg_version}"
+
+if [ -f "${vcpkg_tool_patch}" ]; then
+  git apply --verbose --ignore-whitespace ${vcpkg_tool_patch}
+  echo "Patch successfully applied to the VCPKG tool!"
+fi
+
 if [[ "$OSTYPE" == "msys" ]]; then
   ./bootstrap-vcpkg.bat -disableMetrics
 else
   ./bootstrap-vcpkg.sh -disableMetrics
 fi
-git apply --verbose --ignore-whitespace ${vcpkg_patch}
-echo "Patch successfully applied!"
+
+if [ -f "${vcpkg_ports_patch}" ]; then
+  git apply --verbose --ignore-whitespace ${vcpkg_ports_patch}
+  echo "Patch successfully applied to the VCPKG port files!"
+fi
 
 popd
diff --git a/ci/vcpkg/tool.patch b/ci/vcpkg/tool.patch
new file mode 100644
index 00000000000..49f4efe9218
--- /dev/null
+++ b/ci/vcpkg/tool.patch
@@ -0,0 +1,24 @@
+diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
+index d36f7ef1..01dfcc49 100644
+--- a/scripts/bootstrap.sh
++++ b/scripts/bootstrap.sh
+@@ -123,8 +123,9 @@ fi
+ #   Download and build from source
+ 
+ # Choose the vcpkg binary to download
+-vcpkgDownloadTool="ON"
++vcpkgDownloadTool=${VCPKG_DOWNLOAD_TOOL:-"ON"}
+ vcpkgToolReleaseTag="2021-11-15"
++vcpkgToolSourceReleaseSha="d94b00fd8d0b3abb0379390fc12c9103f209621e899df11c1ccfd9d15558908dbe0cd3e61cf7fe89a28a6a2b42eeb311d74cf1ebcb0f8e8debc671e9be123ec7"
+ if [ "$UNAME" = "Darwin" ]; then
+     echo "Downloading vcpkg-macos..."
+     vcpkgToolReleaseSha="2dc3b9fc8dde1eebdc76e1d40c4e9078d7a892487552349ac75df8e9c33d601830ab06b2283a7ba163de0d70f441574501de6cc1198930a346a3a642e677f93f"
+@@ -223,7 +224,7 @@ else
+         vcpkgCheckEqualFileHash "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha"
+     else
+         echo "Downloading vcpkg tool sources"
+-        vcpkgDownloadFile "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha"
++        vcpkgDownloadFile "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolSourceReleaseSha"
+     fi
+ 
+     echo "Building vcpkg-tool..."

From 31cd6aecf7760921f16408101ce332d70a4483f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 16:26:07 +0100
Subject: [PATCH 10/33] Install glibc 2.18 for vcpkg in manylinux2010

---
 ...=> python-wheel-manylinux-2010.dockerfile} | 43 ++++----
 .../python-wheel-manylinux-2014.dockerfile    | 97 +++++++++++++++++++
 ci/scripts/install_ccache.sh                  | 45 +++++++++
 ci/scripts/install_cmake.sh                   | 42 ++++++++
 ci/scripts/install_glibc.sh                   | 42 ++++++++
 ci/scripts/install_ninja.sh                   | 40 ++++++++
 docker-compose.yml                            |  8 +-
 7 files changed, 288 insertions(+), 29 deletions(-)
 rename ci/docker/{python-wheel-manylinux-201x.dockerfile => python-wheel-manylinux-2010.dockerfile} (73%)
 create mode 100644 ci/docker/python-wheel-manylinux-2014.dockerfile
 create mode 100755 ci/scripts/install_ccache.sh
 create mode 100755 ci/scripts/install_cmake.sh
 create mode 100755 ci/scripts/install_glibc.sh
 create mode 100755 ci/scripts/install_ninja.sh

diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-2010.dockerfile
similarity index 73%
rename from ci/docker/python-wheel-manylinux-201x.dockerfile
rename to ci/docker/python-wheel-manylinux-2010.dockerfile
index 6e1120f2e8f..13b5b67d776 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-2010.dockerfile
@@ -15,52 +15,45 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG base
-FROM ${base}
-
 ARG arch_alias
-ARG arch_short_alias
+FROM quay.io/pypa/manylinux2010_${arch_alias}:2021-10-11-14ac00e
 
 RUN yum install -y git flex curl autoconf zip wget
 
+# Install more recent glibc>2.15 for prebuilt vcpkg binary
+ARG glibc=2.18
+COPY ci/scripts/install_glibc.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc}
+
 # Install CMake
 # AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
+ARG arch
 ARG cmake=3.21.4
-RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \
-    tar -xzf - --directory /usr/local --strip-components=1
+COPY ci/scripts/install_cmake.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local
 
 # Install Ninja
 ARG ninja=1.10.2
-RUN mkdir /tmp/ninja && \
-    wget -q https://github.com/ninja-build/ninja/archive/v${ninja}.tar.gz -O - | \
-    tar -xzf - --directory /tmp/ninja --strip-components=1 && \
-    cd /tmp/ninja && \
-    ./configure.py --bootstrap && \
-    mv ninja /usr/local/bin && \
-    rm -rf /tmp/ninja
+COPY ci/scripts/install_ninja.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ninja.sh ${ninja} /usr/local
 
 # Install ccache
 ARG ccache=4.1
-RUN mkdir /tmp/ccache && \
-    wget -q https://github.com/ccache/ccache/archive/v${ccache}.tar.gz -O - | \
-    tar -xzf - --directory /tmp/ccache --strip-components=1 && \
-    cd /tmp/ccache && \
-    mkdir build && \
-    cd build && \
-    cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON .. && \
-    ninja install && \
-    rm -rf /tmp/ccache
+COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
 
 # Install vcpkg
-ARG vcpkg \
-    vcpkg_download_tool="ON"
+ARG vcpkg
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
      arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
-RUN VCPKG_DOWNLOAD_TOOL=${vcpkg_download_tool} arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg}
+RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
+    patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
+    patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg
 ENV PATH="/opt/vcpkg:${PATH}"
 
+ARG arch_short_alias
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_FORCE_SYSTEM_BINARIES=1 \
diff --git a/ci/docker/python-wheel-manylinux-2014.dockerfile b/ci/docker/python-wheel-manylinux-2014.dockerfile
new file mode 100644
index 00000000000..948d62c863f
--- /dev/null
+++ b/ci/docker/python-wheel-manylinux-2014.dockerfile
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG arch_alias
+FROM quay.io/pypa/manylinux2014_${arch_alias}:2021-10-11-14ac00e
+
+RUN yum install -y git flex curl autoconf zip wget
+
+# Install CMake
+# AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
+ARG arch
+ARG cmake=3.21.4
+COPY ci/scripts/install_cmake.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local
+
+# Install Ninja
+ARG ninja=1.10.2
+COPY ci/scripts/install_ninja.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ninja.sh ${ninja} /usr/local
+
+# Install ccache
+ARG ccache=4.1
+COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
+
+# Install vcpkg
+ARG vcpkg
+COPY ci/vcpkg/*.patch \
+     ci/vcpkg/*linux*.cmake \
+     arrow/ci/vcpkg/
+COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg}
+ENV PATH="/opt/vcpkg:${PATH}"
+
+ARG arch_short_alias
+ARG build_type=release
+ENV CMAKE_BUILD_TYPE=${build_type} \
+    VCPKG_FORCE_SYSTEM_BINARIES=1 \
+    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
+    VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \
+    VCPKG_FEATURE_FLAGS=-manifests
+
+# Need to install the boost-build prior installing the boost packages, otherwise
+# vcpkg will raise an error.
+# TODO(kszucs): factor out the package enumeration to a text file and reuse it
+# from the windows image and potentially in a future macos wheel build
+RUN vcpkg install --clean-after-build \
+        boost-build:${arch_short_alias}-linux && \
+    vcpkg install --clean-after-build \
+        abseil \
+        aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
+        boost-filesystem \
+        brotli \
+        bzip2 \
+        c-ares \
+        curl \
+        flatbuffers \
+        gflags \
+        glog \
+        google-cloud-cpp[core,storage] \
+        grpc \
+        lz4 \
+        openssl \
+        orc \
+        protobuf \
+        rapidjson \
+        re2 \
+        snappy \
+        thrift \
+        utf8proc \
+        zlib \
+        zstd
+
+# ARG python=3.6
+# ENV PYTHON_VERSION=${python}
+# RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \
+#     echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
+
+# SHELL ["/bin/bash", "-i", "-c"]
+# ENTRYPOINT ["/bin/bash", "-i", "-c"]
+
+# COPY python/requirements-wheel-build.txt /arrow/python/
+# RUN pip install -r /arrow/python/requirements-wheel-build.txt
diff --git a/ci/scripts/install_ccache.sh b/ci/scripts/install_ccache.sh
new file mode 100755
index 00000000000..8c64fe56c41
--- /dev/null
+++ b/ci/scripts/install_ccache.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <prefix>"
+  exit 1
+fi
+
+version=$1
+prefix=$2
+url="https://github.com/ccache/ccache/archive/v${version}.tar.gz"
+
+mkdir /tmp/ccache
+wget -q ${url} -O - | tar -xzf - --directory /tmp/ccache --strip-components=1
+
+mkdir /tmp/ccache/build
+pushd /tmp/ccache/build
+cmake \
+  -GNinja \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX=${prefix} \
+  -DZSTD_FROM_INTERNET=ON \
+  ..
+ninja install
+popd
+
+rm -rf /tmp/ccache
diff --git a/ci/scripts/install_cmake.sh b/ci/scripts/install_cmake.sh
new file mode 100755
index 00000000000..2f5e5d52051
--- /dev/null
+++ b/ci/scripts/install_cmake.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+declare -A archs
+archs=([amd64]=x86_64
+       [arm64v8]=aarch64)
+
+declare -A platforms
+platforms=([linux]=linux
+           [macos]=macos
+           [windows]=windows)
+
+if [ "$#" -ne 4 ]; then
+  echo "Usage: $0 <architecture> <platform> <version> <prefix>"
+  exit 1
+fi
+
+arch=${archs[$1]}
+platform=${platforms[$2]}
+version=$3
+prefix=$4
+
+url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-${arch}.tar.gz"
+wget -q ${url} -O - | tar -xzf - --directory ${prefix} --strip-components=1
diff --git a/ci/scripts/install_glibc.sh b/ci/scripts/install_glibc.sh
new file mode 100755
index 00000000000..aecd2893a78
--- /dev/null
+++ b/ci/scripts/install_glibc.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <prefix>"
+  exit 1
+fi
+
+version=$1
+prefix=$2
+
+url="http://ftp.gnu.org/gnu/glibc/glibc-${version}.tar.gz"
+
+mkdir /tmp/glibc
+wget -q ${url} -O - | tar -xzf - --directory /tmp/glibc --strip-components=1
+
+mkdir /tmp/glibc/build
+pushd /tmp/glibc/build
+../configure --prefix=${prefix}
+make -j4
+make install
+popd
+
+rm -rf /tmp/glibc
diff --git a/ci/scripts/install_ninja.sh b/ci/scripts/install_ninja.sh
new file mode 100755
index 00000000000..0440d563fb1
--- /dev/null
+++ b/ci/scripts/install_ninja.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ "$#" -ne 2 ]; then
+  echo "Usage: $0 <version> <prefix>"
+  exit 1
+fi
+
+version=$1
+prefix=$2
+
+url="https://github.com/ninja-build/ninja/archive/v${version}.tar.gz"
+
+mkdir /tmp/ninja
+wget -q ${url} -O - | tar -xzf - --directory /tmp/ninja --strip-components=1
+
+pushd /tmp/ninja
+./configure.py --bootstrap
+mv ninja ${prefix}/bin
+popd
+
+rm -rf /tmp/ninja
diff --git a/docker-compose.yml b/docker-compose.yml
index 8751a8a2972..4cd9250ee0a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -778,13 +778,13 @@ services:
     image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2010-vcpkg-${VCPKG}
     build:
       args:
+        arch: ${ARCH}
         arch_alias: ${ARCH_ALIAS}
         arch_short_alias: ${ARCH_SHORT_ALIAS}
-        base: quay.io/pypa/manylinux2010_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}
       context: .
-      dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
+      dockerfile: ci/docker/python-wheel-manylinux-2010.dockerfile
       cache_from:
         - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2010-vcpkg-${VCPKG}
     environment:
@@ -801,13 +801,13 @@ services:
     image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
     build:
       args:
+        arch: ${ARCH}
         arch_alias: ${ARCH_ALIAS}
         arch_short_alias: ${ARCH_SHORT_ALIAS}
-        base:  quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}
       context: .
-      dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
+      dockerfile: ci/docker/python-wheel-manylinux-2014.dockerfile
       cache_from:
         - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
     environment:

From cbb82b6aac67ac928716b75a2d02afad7d7fc663 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 16:40:15 +0100
Subject: [PATCH 11/33] Remove force compile workaround for vcpkg tool
 installation

---
 .../python-wheel-manylinux-2014.dockerfile    | 16 ++++++-------
 ci/scripts/install_vcpkg.sh                   |  9 +------
 ci/vcpkg/tool.patch                           | 24 -------------------
 3 files changed, 9 insertions(+), 40 deletions(-)
 delete mode 100644 ci/vcpkg/tool.patch

diff --git a/ci/docker/python-wheel-manylinux-2014.dockerfile b/ci/docker/python-wheel-manylinux-2014.dockerfile
index 948d62c863f..4333cf1bba9 100644
--- a/ci/docker/python-wheel-manylinux-2014.dockerfile
+++ b/ci/docker/python-wheel-manylinux-2014.dockerfile
@@ -85,13 +85,13 @@ RUN vcpkg install --clean-after-build \
         zlib \
         zstd
 
-# ARG python=3.6
-# ENV PYTHON_VERSION=${python}
-# RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \
-#     echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
+ARG python=3.6
+ENV PYTHON_VERSION=${python}
+RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \
+    echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
 
-# SHELL ["/bin/bash", "-i", "-c"]
-# ENTRYPOINT ["/bin/bash", "-i", "-c"]
+SHELL ["/bin/bash", "-i", "-c"]
+ENTRYPOINT ["/bin/bash", "-i", "-c"]
 
-# COPY python/requirements-wheel-build.txt /arrow/python/
-# RUN pip install -r /arrow/python/requirements-wheel-build.txt
+COPY python/requirements-wheel-build.txt /arrow/python/
+RUN pip install -r /arrow/python/requirements-wheel-build.txt
diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index a97af435a70..33a644e94ab 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -20,19 +20,17 @@
 set -e
 
 if [ "$#" -lt 1 ]; then
-  echo "Usage: $0 ``<target-directory> [<vcpkg-version> [[<vcpkg-ports-patch> [<vcpkg-tool-patch>]]]"
+  echo "Usage: $0 ``<target-directory> [<vcpkg-version> [<vcpkg-ports-patch>]]"
   exit 1
 fi
 
 arrow_dir=$(readlink -f $(dirname "${0}")/../..)
 default_vcpkg_version=$(cat "${arrow_dir}/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
-default_vcpkg_tool_patch="${arrow_dir}/ci/vcpkg/tool.patch"
 default_vcpkg_ports_patch="${arrow_dir}/ci/vcpkg/ports.patch"
 
 vcpkg_destination=$1
 vcpkg_version=${2:-$default_vcpkg_version}
 vcpkg_ports_patch=${3:-$default_vcpkg_ports_patch}
-vcpkg_tool_patch=${4:-$default_vcpkg_tool_patch}
 
 # reduce the fetched data using a shallow clone
 git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}
@@ -41,11 +39,6 @@ pushd ${vcpkg_destination}
 
 git checkout "${vcpkg_version}"
 
-if [ -f "${vcpkg_tool_patch}" ]; then
-  git apply --verbose --ignore-whitespace ${vcpkg_tool_patch}
-  echo "Patch successfully applied to the VCPKG tool!"
-fi
-
 if [[ "$OSTYPE" == "msys" ]]; then
   ./bootstrap-vcpkg.bat -disableMetrics
 else
diff --git a/ci/vcpkg/tool.patch b/ci/vcpkg/tool.patch
deleted file mode 100644
index 49f4efe9218..00000000000
--- a/ci/vcpkg/tool.patch
+++ /dev/null
@@ -1,24 +0,0 @@
-diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh
-index d36f7ef1..01dfcc49 100644
---- a/scripts/bootstrap.sh
-+++ b/scripts/bootstrap.sh
-@@ -123,8 +123,9 @@ fi
- #   Download and build from source
- 
- # Choose the vcpkg binary to download
--vcpkgDownloadTool="ON"
-+vcpkgDownloadTool=${VCPKG_DOWNLOAD_TOOL:-"ON"}
- vcpkgToolReleaseTag="2021-11-15"
-+vcpkgToolSourceReleaseSha="d94b00fd8d0b3abb0379390fc12c9103f209621e899df11c1ccfd9d15558908dbe0cd3e61cf7fe89a28a6a2b42eeb311d74cf1ebcb0f8e8debc671e9be123ec7"
- if [ "$UNAME" = "Darwin" ]; then
-     echo "Downloading vcpkg-macos..."
-     vcpkgToolReleaseSha="2dc3b9fc8dde1eebdc76e1d40c4e9078d7a892487552349ac75df8e9c33d601830ab06b2283a7ba163de0d70f441574501de6cc1198930a346a3a642e677f93f"
-@@ -223,7 +224,7 @@ else
-         vcpkgCheckEqualFileHash "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha"
-     else
-         echo "Downloading vcpkg tool sources"
--        vcpkgDownloadFile "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolReleaseSha"
-+        vcpkgDownloadFile "$vcpkgToolUrl" "$tarballPath" "$vcpkgToolSourceReleaseSha"
-     fi
- 
-     echo "Building vcpkg-tool..."

From 0cf9cb7d00b7a8c03b0435c7f9235046f5395f87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 16:46:44 +0100
Subject: [PATCH 12/33] Install cmake=3.21.4 in the amd64 macOS builds

---
 dev/tasks/python-wheels/github.osx.amd64.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index ae72c61d1db..b0c433359ed 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -40,7 +40,10 @@ jobs:
       {{ macros.github_checkout_arrow()|indent }}
 
       - name: Install System Dependencies
-        run: brew install bash bison coreutils ninja cmake
+        run: brew install bash bison coreutils ninja
+
+      - name: Install Specific CMake version
+        uses: lukka/get-cmake@v3.21.4
 
       - name: Retrieve VCPKG version from arrow/.env
         shell: bash

From 3dd52ec757920999d8ed7cd92f735863e0c1c30c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 17:07:08 +0100
Subject: [PATCH 13/33] Portable retrieval of arrow's absolute path in bash

---
 ci/scripts/install_vcpkg.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index 33a644e94ab..a5726833e3c 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -24,7 +24,7 @@ if [ "$#" -lt 1 ]; then
   exit 1
 fi
 
-arrow_dir=$(readlink -f $(dirname "${0}")/../..)
+arrow_dir=$(cd -- "$(dirname -- "$0")" && pwd -P)
 default_vcpkg_version=$(cat "${arrow_dir}/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
 default_vcpkg_ports_patch="${arrow_dir}/ci/vcpkg/ports.patch"
 

From a01e3de3aa4c923042413751741a3598f335a04e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 17:16:21 +0100
Subject: [PATCH 14/33] One level upper

---
 ci/scripts/install_vcpkg.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
index a5726833e3c..cc80582326e 100755
--- a/ci/scripts/install_vcpkg.sh
+++ b/ci/scripts/install_vcpkg.sh
@@ -24,7 +24,7 @@ if [ "$#" -lt 1 ]; then
   exit 1
 fi
 
-arrow_dir=$(cd -- "$(dirname -- "$0")" && pwd -P)
+arrow_dir=$(cd -- "$(dirname -- "$0")/../.." && pwd -P)
 default_vcpkg_version=$(cat "${arrow_dir}/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
 default_vcpkg_ports_patch="${arrow_dir}/ci/vcpkg/ports.patch"
 

From a7367297a9000522dd2da9ad9f620c5ffccd3c66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 17:22:30 +0100
Subject: [PATCH 15/33] Patch aws-sdk-cpp port to build on arm

---
 ci/vcpkg/ports.patch                         | 13 +++++++++++++
 dev/tasks/python-wheels/github.osx.arm64.yml |  8 +++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index 1ce7d3d8e58..250af4055be 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -1,3 +1,16 @@
+diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
+index 3836e2b6..cd88ef07 100644
+--- a/ports/aws-sdk-cpp/vcpkg.json
++++ b/ports/aws-sdk-cpp/vcpkg.json
+@@ -5,7 +5,7 @@
+   "port-version": 5,
+   "description": "AWS SDK for C++",
+   "homepage": "https://github.com/aws/aws-sdk-cpp",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-crt-cpp",
+     {
 diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
 index ce8b7570..8255ba24 100644
 --- a/ports/curl/portfile.cmake
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 29ca3995e68..63353f81050 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -55,11 +55,17 @@ jobs:
         shell: bash
         run: echo "/opt/homebrew/opt/bison/bin" >> $GITHUB_PATH
 
+      - name: Retrieve VCPKG version from arrow/.env
+        shell: bash
+        run: |
+          vcpkg_version=$(cat "arrow/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
+          echo "VCPKG_VERSION=$vcpkg_version" >> $GITHUB_ENV
+
       - name: Install Vcpkg
         shell: bash
         env:
           MACOSX_DEPLOYMENT_TARGET: "11.0"
-        run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT
+        run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
       - name: Install OpenSSL
         shell: bash

From 4108d9adbcba3d88659e61e6f3d6e5c9514244ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 17:24:07 +0100
Subject: [PATCH 16/33] Pin an earlier cmake for macos GHA job

---
 dev/tasks/python-wheels/github.osx.amd64.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index b0c433359ed..bc94bbc3b2c 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -43,7 +43,7 @@ jobs:
         run: brew install bash bison coreutils ninja
 
       - name: Install Specific CMake version
-        uses: lukka/get-cmake@v3.21.4
+        uses: lukka/get-cmake@v3.21.2
 
       - name: Retrieve VCPKG version from arrow/.env
         shell: bash

From 55b57aa9b425bd3227116a9951e1b7249ae861e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 17:37:34 +0100
Subject: [PATCH 17/33] Enable arm for all aws vcpkg ports

---
 ci/vcpkg/ports.patch | 142 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 142 insertions(+)

diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index 250af4055be..34d1fcb7e17 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -1,3 +1,145 @@
+diff --git a/ports/aws-c-auth/vcpkg.json b/ports/aws-c-auth/vcpkg.json
+index dc8f75e8..be703324 100644
+--- a/ports/aws-c-auth/vcpkg.json
++++ b/ports/aws-c-auth/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "C99 library implementation of AWS client-side authentication: standard credentials providers and signing.",
+   "homepage": "https://github.com/awslabs/aws-c-auth",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-common",
+     "aws-c-http",
+diff --git a/ports/aws-c-cal/vcpkg.json b/ports/aws-c-cal/vcpkg.json
+index 48c7406c..40e284dd 100644
+--- a/ports/aws-c-cal/vcpkg.json
++++ b/ports/aws-c-cal/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "C99 wrapper for cryptography primitives.",
+   "homepage": "https://github.com/awslabs/aws-c-cal",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-common",
+     {
+diff --git a/ports/aws-c-common/vcpkg.json b/ports/aws-c-common/vcpkg.json
+index 67d58320..3c8e96f7 100644
+--- a/ports/aws-c-common/vcpkg.json
++++ b/ports/aws-c-common/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "AWS common library for C",
+   "homepage": "https://github.com/awslabs/aws-c-common",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "openssl",
+     {
+diff --git a/ports/aws-c-compression/vcpkg.json b/ports/aws-c-compression/vcpkg.json
+index 8b62d038..7aa75eda 100644
+--- a/ports/aws-c-compression/vcpkg.json
++++ b/ports/aws-c-compression/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "C99 implementation of huffman encoding/decoding",
+   "homepage": "https://github.com/awslabs/aws-c-compression",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-common",
+     {
+diff --git a/ports/aws-c-event-stream/vcpkg.json b/ports/aws-c-event-stream/vcpkg.json
+index 7018ea0b..df3f85af 100644
+--- a/ports/aws-c-event-stream/vcpkg.json
++++ b/ports/aws-c-event-stream/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "C99 implementation of the vnd.amazon.event-stream content-type.",
+   "homepage": "https://github.com/awslabs/aws-c-event-stream",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-cal",
+     "aws-c-common",
+diff --git a/ports/aws-c-http/vcpkg.json b/ports/aws-c-http/vcpkg.json
+index f7cf6547..20ee8be7 100644
+--- a/ports/aws-c-http/vcpkg.json
++++ b/ports/aws-c-http/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "C99 implementation of the HTTP/1.1 and HTTP/2 specifications",
+   "homepage": "https://github.com/awslabs/aws-c-http",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-cal",
+     "aws-c-common",
+diff --git a/ports/aws-c-io/vcpkg.json b/ports/aws-c-io/vcpkg.json
+index 60a1fb93..b695deb5 100644
+--- a/ports/aws-c-io/vcpkg.json
++++ b/ports/aws-c-io/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "Handles all IO and TLS work for application protocols.",
+   "homepage": "https://github.com/awslabs/aws-c-io",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-cal",
+     "aws-c-common",
+diff --git a/ports/aws-c-mqtt/vcpkg.json b/ports/aws-c-mqtt/vcpkg.json
+index de7e464e..3f459a79 100644
+--- a/ports/aws-c-mqtt/vcpkg.json
++++ b/ports/aws-c-mqtt/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "C99 implementation of the MQTT 3.1.1 specification.",
+   "homepage": "https://github.com/awslabs/aws-c-mqtt",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     {
+       "name": "s2n",
+diff --git a/ports/aws-c-s3/vcpkg.json b/ports/aws-c-s3/vcpkg.json
+index 31dba925..902c07f9 100644
+--- a/ports/aws-c-s3/vcpkg.json
++++ b/ports/aws-c-s3/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 1,
+   "description": "C99 library implementation for communicating with the S3 service, designed for maximizing throughput on high bandwidth EC2 instances.",
+   "homepage": "https://github.com/awslabs/aws-c-s3",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-auth",
+     "aws-c-cal",
+diff --git a/ports/aws-checksums/vcpkg.json b/ports/aws-checksums/vcpkg.json
+index 3cdea453..1cbed512 100644
+--- a/ports/aws-checksums/vcpkg.json
++++ b/ports/aws-checksums/vcpkg.json
+@@ -4,7 +4,6 @@
+   "port-version": 2,
+   "description": "Cross-Platform HW accelerated CRC32c and CRC32 with fallback to efficient SW implementations.",
+   "homepage": "https://github.com/awslabs/aws-checksums",
+-  "supports": "!arm",
+   "dependencies": [
+     "aws-c-common",
+     {
+diff --git a/ports/aws-crt-cpp/vcpkg.json b/ports/aws-crt-cpp/vcpkg.json
+index 3b8cfafd..38c37ec9 100644
+--- a/ports/aws-crt-cpp/vcpkg.json
++++ b/ports/aws-crt-cpp/vcpkg.json
+@@ -4,7 +4,7 @@
+   "port-version": 2,
+   "description": "Cross-Platform HW accelerated CRC32c and CRC32 with fallback to efficient SW implementations.",
+   "homepage": "https://github.com/awslabs/aws-crt-cpp",
+-  "supports": "!arm & !uwp",
++  "supports": "!uwp",
+   "dependencies": [
+     "aws-c-auth",
+     "aws-c-cal",
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json

From 6f1822d7c233c159c638ff16b585187a2e23b0ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Thu, 25 Nov 2021 17:47:08 +0100
Subject: [PATCH 18/33] Invalidate gha cache for amd64 macos

---
 dev/tasks/python-wheels/github.osx.amd64.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index bc94bbc3b2c..b5d51e210c7 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -55,7 +55,7 @@ jobs:
         id: vcpkg-cache
         with:
           path: vcpkg
-          key: vcpkg-{{ macos_deployment_target }}-{{ "${{ env.VCPKG_VERSION }}" }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}
+          key: vcpkg-{{ macos_deployment_target }}-{{ "${{ env.VCPKG_VERSION }}" }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}-1
 
       - name: Install Vcpkg
         if: steps.vcpkg-cache.outputs.cache-hit != 'true'

From 796f0b530583f36e3c55660aa4f72e482a265cd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 02:36:17 +0100
Subject: [PATCH 19/33] Fix AWS ADK <=> libcrypt linking error

---
 .../python-wheel-manylinux-2014.dockerfile    | 97 -------------------
 ...=> python-wheel-manylinux-201x.dockerfile} | 33 ++++---
 ci/scripts/python_wheel_manylinux_build.sh    |  2 +-
 ...g.cmake => amd64-linux-static-debug.cmake} |  0
 ...cmake => amd64-linux-static-release.cmake} |  0
 ...bug.cmake => amd64-osx-static-debug.cmake} |  0
 ...e.cmake => amd64-osx-static-release.cmake} |  0
 ...ke => amd64-windows-static-md-debug.cmake} |  0
 ... => amd64-windows-static-md-release.cmake} |  0
 ci/vcpkg/ports.patch                          | 38 ++++++++
 docker-compose.yml                            | 14 ++-
 11 files changed, 63 insertions(+), 121 deletions(-)
 delete mode 100644 ci/docker/python-wheel-manylinux-2014.dockerfile
 rename ci/docker/{python-wheel-manylinux-2010.dockerfile => python-wheel-manylinux-201x.dockerfile} (81%)
 rename ci/vcpkg/{x64-linux-static-debug.cmake => amd64-linux-static-debug.cmake} (100%)
 rename ci/vcpkg/{x64-linux-static-release.cmake => amd64-linux-static-release.cmake} (100%)
 rename ci/vcpkg/{x64-osx-static-debug.cmake => amd64-osx-static-debug.cmake} (100%)
 rename ci/vcpkg/{x64-osx-static-release.cmake => amd64-osx-static-release.cmake} (100%)
 rename ci/vcpkg/{x64-windows-static-md-debug.cmake => amd64-windows-static-md-debug.cmake} (100%)
 rename ci/vcpkg/{x64-windows-static-md-release.cmake => amd64-windows-static-md-release.cmake} (100%)

diff --git a/ci/docker/python-wheel-manylinux-2014.dockerfile b/ci/docker/python-wheel-manylinux-2014.dockerfile
deleted file mode 100644
index 4333cf1bba9..00000000000
--- a/ci/docker/python-wheel-manylinux-2014.dockerfile
+++ /dev/null
@@ -1,97 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-ARG arch_alias
-FROM quay.io/pypa/manylinux2014_${arch_alias}:2021-10-11-14ac00e
-
-RUN yum install -y git flex curl autoconf zip wget
-
-# Install CMake
-# AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
-ARG arch
-ARG cmake=3.21.4
-COPY ci/scripts/install_cmake.sh arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local
-
-# Install Ninja
-ARG ninja=1.10.2
-COPY ci/scripts/install_ninja.sh arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_ninja.sh ${ninja} /usr/local
-
-# Install ccache
-ARG ccache=4.1
-COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
-
-# Install vcpkg
-ARG vcpkg
-COPY ci/vcpkg/*.patch \
-     ci/vcpkg/*linux*.cmake \
-     arrow/ci/vcpkg/
-COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
-RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg}
-ENV PATH="/opt/vcpkg:${PATH}"
-
-ARG arch_short_alias
-ARG build_type=release
-ENV CMAKE_BUILD_TYPE=${build_type} \
-    VCPKG_FORCE_SYSTEM_BINARIES=1 \
-    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
-    VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \
-    VCPKG_FEATURE_FLAGS=-manifests
-
-# Need to install the boost-build prior installing the boost packages, otherwise
-# vcpkg will raise an error.
-# TODO(kszucs): factor out the package enumeration to a text file and reuse it
-# from the windows image and potentially in a future macos wheel build
-RUN vcpkg install --clean-after-build \
-        boost-build:${arch_short_alias}-linux && \
-    vcpkg install --clean-after-build \
-        abseil \
-        aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
-        boost-filesystem \
-        brotli \
-        bzip2 \
-        c-ares \
-        curl \
-        flatbuffers \
-        gflags \
-        glog \
-        google-cloud-cpp[core,storage] \
-        grpc \
-        lz4 \
-        openssl \
-        orc \
-        protobuf \
-        rapidjson \
-        re2 \
-        snappy \
-        thrift \
-        utf8proc \
-        zlib \
-        zstd
-
-ARG python=3.6
-ENV PYTHON_VERSION=${python}
-RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \
-    echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
-
-SHELL ["/bin/bash", "-i", "-c"]
-ENTRYPOINT ["/bin/bash", "-i", "-c"]
-
-COPY python/requirements-wheel-build.txt /arrow/python/
-RUN pip install -r /arrow/python/requirements-wheel-build.txt
diff --git a/ci/docker/python-wheel-manylinux-2010.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
similarity index 81%
rename from ci/docker/python-wheel-manylinux-2010.dockerfile
rename to ci/docker/python-wheel-manylinux-201x.dockerfile
index 13b5b67d776..9a66e869b56 100644
--- a/ci/docker/python-wheel-manylinux-2010.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -15,15 +15,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG arch_alias
-FROM quay.io/pypa/manylinux2010_${arch_alias}:2021-10-11-14ac00e
+ARG base
+FROM ${base}
 
-RUN yum install -y git flex curl autoconf zip wget
+ARG manylinux
+ENV MANYLINUX_VERSION=${manylinux}
 
-# Install more recent glibc>2.15 for prebuilt vcpkg binary
-ARG glibc=2.18
-COPY ci/scripts/install_glibc.sh arrow/ci/scripts/
-RUN /arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc}
+# Install basic dependencies
+RUN yum install -y git flex curl autoconf zip wget
 
 # Install CMake
 # AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
@@ -42,23 +41,29 @@ ARG ccache=4.1
 COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
 
-# Install vcpkg
+# Install vcpkg and in case of manylinux2010 install a more recent glibc>2.15
+# for the prebuilt vcpkg binary
 ARG vcpkg
+ARG glibc=2.18
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
      arrow/ci/vcpkg/
-COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
+COPY ci/scripts/install_vcpkg.sh \
+     ci/scripts/install_glibc.sh \
+     arrow/ci/scripts/
 RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
-    patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
-    patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg
+    if [ "${manylinux}" == "2010" ]; then \
+        arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
+        patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
+        patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
+    fi
 ENV PATH="/opt/vcpkg:${PATH}"
 
-ARG arch_short_alias
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_FORCE_SYSTEM_BINARIES=1 \
     VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
-    VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \
+    VCPKG_DEFAULT_TRIPLET=${arch}-linux-static-${build_type} \
     VCPKG_FEATURE_FLAGS=-manifests
 
 # Need to install the boost-build prior installing the boost packages, otherwise
@@ -66,8 +71,6 @@ ENV CMAKE_BUILD_TYPE=${build_type} \
 # TODO(kszucs): factor out the package enumeration to a text file and reuse it
 # from the windows image and potentially in a future macos wheel build
 RUN vcpkg install --clean-after-build \
-        boost-build:${arch_short_alias}-linux && \
-    vcpkg install --clean-after-build \
         abseil \
         aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
         boost-filesystem \
diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
index 434605cf202..938ada84952 100755
--- a/ci/scripts/python_wheel_manylinux_build.sh
+++ b/ci/scripts/python_wheel_manylinux_build.sh
@@ -88,7 +88,7 @@ cmake \
     -DARROW_DATASET=${ARROW_DATASET} \
     -DARROW_DEPENDENCY_SOURCE="VCPKG" \
     -DARROW_DEPENDENCY_USE_SHARED=OFF \
-    -DARROW_FLIGHT==${ARROW_FLIGHT} \
+    -DARROW_FLIGHT=${ARROW_FLIGHT} \
     -DARROW_GANDIVA=${ARROW_GANDIVA} \
     -DARROW_HDFS=${ARROW_HDFS} \
     -DARROW_JEMALLOC=${ARROW_JEMALLOC} \
diff --git a/ci/vcpkg/x64-linux-static-debug.cmake b/ci/vcpkg/amd64-linux-static-debug.cmake
similarity index 100%
rename from ci/vcpkg/x64-linux-static-debug.cmake
rename to ci/vcpkg/amd64-linux-static-debug.cmake
diff --git a/ci/vcpkg/x64-linux-static-release.cmake b/ci/vcpkg/amd64-linux-static-release.cmake
similarity index 100%
rename from ci/vcpkg/x64-linux-static-release.cmake
rename to ci/vcpkg/amd64-linux-static-release.cmake
diff --git a/ci/vcpkg/x64-osx-static-debug.cmake b/ci/vcpkg/amd64-osx-static-debug.cmake
similarity index 100%
rename from ci/vcpkg/x64-osx-static-debug.cmake
rename to ci/vcpkg/amd64-osx-static-debug.cmake
diff --git a/ci/vcpkg/x64-osx-static-release.cmake b/ci/vcpkg/amd64-osx-static-release.cmake
similarity index 100%
rename from ci/vcpkg/x64-osx-static-release.cmake
rename to ci/vcpkg/amd64-osx-static-release.cmake
diff --git a/ci/vcpkg/x64-windows-static-md-debug.cmake b/ci/vcpkg/amd64-windows-static-md-debug.cmake
similarity index 100%
rename from ci/vcpkg/x64-windows-static-md-debug.cmake
rename to ci/vcpkg/amd64-windows-static-md-debug.cmake
diff --git a/ci/vcpkg/x64-windows-static-md-release.cmake b/ci/vcpkg/amd64-windows-static-md-release.cmake
similarity index 100%
rename from ci/vcpkg/x64-windows-static-md-release.cmake
rename to ci/vcpkg/amd64-windows-static-md-release.cmake
diff --git a/ci/vcpkg/ports.patch b/ci/vcpkg/ports.patch
index 34d1fcb7e17..940722abb2e 100644
--- a/ci/vcpkg/ports.patch
+++ b/ci/vcpkg/ports.patch
@@ -140,6 +140,44 @@ index 3b8cfafd..38c37ec9 100644
    "dependencies": [
      "aws-c-auth",
      "aws-c-cal",
+diff --git a/ports/aws-sdk-cpp/fix-find-crypto.patch b/ports/aws-sdk-cpp/fix-find-crypto.patch
+new file mode 100644
+index 00000000..4566cc66
+--- /dev/null
++++ b/ports/aws-sdk-cpp/fix-find-crypto.patch
+@@ -0,0 +1,20 @@
++diff --git a/cmake/setup_cmake_find_module.cmake b/cmake/setup_cmake_find_module.cmake
++index f5f147a0f..4561b8c39 100644
++--- a/cmake/setup_cmake_find_module.cmake
+++++ b/cmake/setup_cmake_find_module.cmake
++@@ -30,15 +30,6 @@ file(WRITE
++     "set(AWSSDK_INSTALL_BINDIR ${BINARY_DIRECTORY})\n"
++     "set(AWSSDK_INSTALL_INCLUDEDIR ${INCLUDE_DIRECTORY})\n"
++     "set(AWSSDK_INSTALL_ARCHIVEDIR ${ARCHIVE_DIRECTORY})\n"
++-    "if (NOT LibCrypto_INCLUDE_DIR)\n"
++-    "    set(LibCrypto_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR} CACHE INTERNAL \"The OpenSSL include directory\")\n"
++-    "endif()\n"
++-    "if (NOT LibCrypto_STATIC_LIBRARY)\n"
++-    "    set(LibCrypto_STATIC_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto static library\")\n"
++-    "endif()\n"
++-    "if (NOT LibCrypto_SHARED_LIBRARY)\n"
++-    "    set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
++-    "endif()\n"
++     )
++ 
++ if (NOT SIMPLE_INSTALL)
+diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
+index 2d6bba4d..0ac47887 100644
+--- a/ports/aws-sdk-cpp/portfile.cmake
++++ b/ports/aws-sdk-cpp/portfile.cmake
+@@ -8,6 +8,7 @@ vcpkg_from_github(
+     PATCHES
+         patch-relocatable-rpath.patch
+         fix-aws-root.patch
++        fix-find-crypto.patch
+ )
+ 
+ string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/aws-sdk-cpp/vcpkg.json b/ports/aws-sdk-cpp/vcpkg.json
 index 3836e2b6..cd88ef07 100644
 --- a/ports/aws-sdk-cpp/vcpkg.json
diff --git a/docker-compose.yml b/docker-compose.yml
index 4cd9250ee0a..7731518de58 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -779,17 +779,16 @@ services:
     build:
       args:
         arch: ${ARCH}
-        arch_alias: ${ARCH_ALIAS}
-        arch_short_alias: ${ARCH_SHORT_ALIAS}
+        base: quay.io/pypa/manylinux2010_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}
+        manylinux: 2010
       context: .
-      dockerfile: ci/docker/python-wheel-manylinux-2010.dockerfile
+      dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
       cache_from:
         - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2010-vcpkg-${VCPKG}
     environment:
       <<: *ccache
-      MANYLINUX_VERSION: 2010
     volumes:
       - .:/arrow:delegated
       - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2010-ccache:/ccache:delegated
@@ -802,17 +801,16 @@ services:
     build:
       args:
         arch: ${ARCH}
-        arch_alias: ${ARCH_ALIAS}
-        arch_short_alias: ${ARCH_SHORT_ALIAS}
+        base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}
+        manylinux: 2014
       context: .
-      dockerfile: ci/docker/python-wheel-manylinux-2014.dockerfile
+      dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
       cache_from:
         - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
     environment:
       <<: *ccache
-      MANYLINUX_VERSION: 2014
     volumes:
       - .:/arrow:delegated
       - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated

From e041fa547f6edd02d517f0beab53a4c03b4e0eab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 02:55:23 +0100
Subject: [PATCH 20/33] Workaround for google-cloud-sdk installation error on
 amd64 macos

---
 dev/tasks/python-wheels/github.linux.amd64.yml | 2 +-
 dev/tasks/python-wheels/github.osx.amd64.yml   | 8 ++++++--
 dev/tasks/python-wheels/github.osx.arm64.yml   | 4 ++--
 dev/tasks/python-wheels/travis.linux.arm64.yml | 2 +-
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml b/dev/tasks/python-wheels/github.linux.amd64.yml
index dc2386482f1..7e079172a0a 100644
--- a/dev/tasks/python-wheels/github.linux.amd64.yml
+++ b/dev/tasks/python-wheels/github.linux.amd64.yml
@@ -47,7 +47,7 @@ jobs:
       {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
       {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
 
-      {% if arrow.branch == 'master' %}
+      {% if True or arrow.branch == 'master' %}
       - name: Push Docker Image
         shell: bash
         run: |
diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.amd64.yml
index b5d51e210c7..88cf85b6ffa 100644
--- a/dev/tasks/python-wheels/github.osx.amd64.yml
+++ b/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -27,7 +27,7 @@ env:
   PYARROW_VERSION: "{{ arrow.no_rc_version }}"
   PYTHON_VERSION: "{{ python_version }}"
   PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}"
-  VCPKG_DEFAULT_TRIPLET: x64-osx-static-release
+  VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release
   VCPKG_FEATURE_FLAGS: "-manifests"
   VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }}
   VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }}
@@ -55,7 +55,7 @@ jobs:
         id: vcpkg-cache
         with:
           path: vcpkg
-          key: vcpkg-{{ macos_deployment_target }}-{{ "${{ env.VCPKG_VERSION }}" }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}-1
+          key: vcpkg-{{ macos_deployment_target }}-{{ "${{ env.VCPKG_VERSION }}" }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}
 
       - name: Install Vcpkg
         if: steps.vcpkg-cache.outputs.cache-hit != 'true'
@@ -64,6 +64,10 @@ jobs:
           MACOSX_DEPLOYMENT_TARGET: "10.15"
         run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
+      # workaround for https://github.com/apache/arrow/pull/11569#issuecomment-979592053
+      - name: Install OpenSSL
+        run: $VCPKG_ROOT/vcpkg install openssl
+
       - name: Install Packages
         run: |
           $VCPKG_ROOT/vcpkg install \
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 63353f81050..487f97bce9c 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -80,12 +80,12 @@ jobs:
       - name: Create universal binaries for OpenSSL
         shell: bash
         run: |
-          for arch in arm64 x64; do
+          for arch in arm64 amd64; do
             VCPKG_DEFAULT_TRIPLET=${arch}-osx-static-release arch -arm64 $VCPKG_ROOT/vcpkg install openssl
           done
           for lib in libcrypto libssl; do
             lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \
-                         $VCPKG_ROOT/installed/x64-osx-static-release/lib/${lib}.a \
+                         $VCPKG_ROOT/installed/amd64-osx-static-release/lib/${lib}.a \
                  -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a
           done
       {% endif %}
diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml b/dev/tasks/python-wheels/travis.linux.arm64.yml
index d32d89d8301..7ee2f164dfd 100644
--- a/dev/tasks/python-wheels/travis.linux.arm64.yml
+++ b/dev/tasks/python-wheels/travis.linux.arm64.yml
@@ -66,7 +66,7 @@ after_success:
   {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }}
   {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }}
 
-  {% if arrow.branch == 'master' %}
+  {% if True or arrow.branch == 'master' %}
   # Push the docker image to dockerhub
   - archery docker push python-wheel-manylinux-{{ manylinux_version }}
   - archery docker push python-wheel-manylinux-test-unittests

From 573c8a0109d2b3bfc42beff61fce89a719fed33c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 03:42:19 +0100
Subject: [PATCH 21/33] Fix arch in triplet naming

---
 .env                                          |  2 +-
 .../python-wheel-manylinux-201x.dockerfile    |  6 +++--
 dev/archery/archery/docker.py                 | 26 ++++++++++---------
 docker-compose.yml                            |  2 ++
 4 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/.env b/.env
index 088742ed570..1682d1d4746 100644
--- a/.env
+++ b/.env
@@ -36,7 +36,7 @@ BUILDKIT_INLINE_CACHE=1
 # different architecture notations
 ARCH=amd64
 ARCH_ALIAS=x86_64
-ARCH_SHORT_ALIAS=x64
+ARCH_SHORT=amd64
 
 ULIMIT_CORE=-1
 REPO=apache/arrow-dev
diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
index 9a66e869b56..dee2b1244b1 100644
--- a/ci/docker/python-wheel-manylinux-201x.dockerfile
+++ b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -18,7 +18,10 @@
 ARG base
 FROM ${base}
 
+ARG arch
+ARG arch_short
 ARG manylinux
+
 ENV MANYLINUX_VERSION=${manylinux}
 
 # Install basic dependencies
@@ -26,7 +29,6 @@ RUN yum install -y git flex curl autoconf zip wget
 
 # Install CMake
 # AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
-ARG arch
 ARG cmake=3.21.4
 COPY ci/scripts/install_cmake.sh arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local
@@ -63,7 +65,7 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_FORCE_SYSTEM_BINARIES=1 \
     VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
-    VCPKG_DEFAULT_TRIPLET=${arch}-linux-static-${build_type} \
+    VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
     VCPKG_FEATURE_FLAGS=-manifests
 
 # Need to install the boost-build prior installing the boost packages, otherwise
diff --git a/dev/archery/archery/docker.py b/dev/archery/archery/docker.py
index 17d4c713afc..cb68a4b5270 100644
--- a/dev/archery/archery/docker.py
+++ b/dev/archery/archery/docker.py
@@ -48,6 +48,18 @@ def _sanitize_command(cmd):
     return re.sub(r"\s+", " ", cmd)
 
 
+_arch_short_mapping = {
+    'amd64': 'amd64',
+    'arm64v8': 'arm64',
+    's390x': 's390x'
+}
+_arch_alias_mapping = {
+    'amd64': 'x86_64',
+    'arm64v8': 'aarch64',
+    's390x': 's390x'
+}
+
+
 class UndefinedImage(Exception):
     pass
 
@@ -82,18 +94,8 @@ def _read_env(self, dotenv_path, params):
 
         # translate docker's architecture notation to a more widely used one
         arch = self.env.get('ARCH', 'amd64')
-        arch_aliases = {
-            'amd64': 'x86_64',
-            'arm64v8': 'aarch64',
-            's390x': 's390x'
-        }
-        arch_short_aliases = {
-            'amd64': 'x64',
-            'arm64v8': 'arm64',
-            's390x': 's390x'
-        }
-        self.env['ARCH_ALIAS'] = arch_aliases.get(arch, arch)
-        self.env['ARCH_SHORT_ALIAS'] = arch_short_aliases.get(arch, arch)
+        self.env['ARCH_ALIAS'] = _arch_alias_mapping.get(arch, arch)
+        self.env['ARCH_SHORT'] = _arch_short_mapping.get(arch, arch)
 
     def _read_config(self, config_path, compose_bin):
         """
diff --git a/docker-compose.yml b/docker-compose.yml
index 7731518de58..328b5978240 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -779,6 +779,7 @@ services:
     build:
       args:
         arch: ${ARCH}
+        arch_short: ${ARCH_SHORT}
         base: quay.io/pypa/manylinux2010_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}
@@ -801,6 +802,7 @@ services:
     build:
       args:
         arch: ${ARCH}
+        arch_short: ${ARCH_SHORT}
         base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2021-10-11-14ac00e
         vcpkg: ${VCPKG}
         python: ${PYTHON}

From ee658ad972a52685b7615c066ff0ee541cd9bb21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 03:56:49 +0100
Subject: [PATCH 22/33] Fix windows dockerfile; force arm64 triplet on M1 mac
 in case of non-universal builds

---
 ci/docker/python-wheel-windows-vs2017.dockerfile |  2 +-
 dev/tasks/python-wheels/github.osx.arm64.yml     | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
index 234ec667918..f7e581daf5b 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -46,7 +46,7 @@ RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
-    VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \
+    VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
     VCPKG_FEATURE_FLAGS=-manifests
 
 RUN vcpkg install --clean-after-build \
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 487f97bce9c..543e22e4c3e 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -67,10 +67,6 @@ jobs:
           MACOSX_DEPLOYMENT_TARGET: "11.0"
         run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
-      - name: Install OpenSSL
-        shell: bash
-        run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl
-
       {% if arch == "universal2" %}
       # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is
       # unable to propagate the list of architectures from VCPKG_OSX_ARCHIETCTURES.
@@ -81,13 +77,17 @@ jobs:
         shell: bash
         run: |
           for arch in arm64 amd64; do
-            VCPKG_DEFAULT_TRIPLET=${arch}-osx-static-release arch -arm64 $VCPKG_ROOT/vcpkg install openssl
+            arch -arm64 $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
           done
           for lib in libcrypto libssl; do
             lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \
                          $VCPKG_ROOT/installed/amd64-osx-static-release/lib/${lib}.a \
                  -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a
           done
+      {% else %}
+      - name: Install OpenSSL
+        shell: bash
+        run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl:arm64-osx-static-release
       {% endif %}
 
       - name: Install Packages

From 0606f745a0ed791ce6a5d464542a1454180cfbcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 04:03:54 +0100
Subject: [PATCH 23/33] Remove unused docker.py

---
 dev/archery/archery/docker.py      | 404 -----------------------------
 dev/archery/archery/docker/core.py |  23 +-
 2 files changed, 11 insertions(+), 416 deletions(-)
 delete mode 100644 dev/archery/archery/docker.py

diff --git a/dev/archery/archery/docker.py b/dev/archery/archery/docker.py
deleted file mode 100644
index cb68a4b5270..00000000000
--- a/dev/archery/archery/docker.py
+++ /dev/null
@@ -1,404 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-import re
-import subprocess
-from io import StringIO
-
-from dotenv import dotenv_values
-from ruamel.yaml import YAML
-
-from .utils.command import Command, default_bin
-from .compat import _ensure_path
-
-
-def flatten(node, parents=None):
-    parents = list(parents or [])
-    if isinstance(node, str):
-        yield (node, parents)
-    elif isinstance(node, list):
-        for value in node:
-            yield from flatten(value, parents=parents)
-    elif isinstance(node, dict):
-        for key, value in node.items():
-            yield (key, parents)
-            yield from flatten(value, parents=parents + [key])
-    else:
-        raise TypeError(node)
-
-
-def _sanitize_command(cmd):
-    if isinstance(cmd, list):
-        cmd = " ".join(cmd)
-    return re.sub(r"\s+", " ", cmd)
-
-
-_arch_short_mapping = {
-    'amd64': 'amd64',
-    'arm64v8': 'arm64',
-    's390x': 's390x'
-}
-_arch_alias_mapping = {
-    'amd64': 'x86_64',
-    'arm64v8': 'aarch64',
-    's390x': 's390x'
-}
-
-
-class UndefinedImage(Exception):
-    pass
-
-
-class ComposeConfig:
-
-    def __init__(self, config_path, dotenv_path, compose_bin, params=None):
-        config_path = _ensure_path(config_path)
-        if dotenv_path:
-            dotenv_path = _ensure_path(dotenv_path)
-        else:
-            dotenv_path = config_path.parent / '.env'
-        self._read_env(dotenv_path, params)
-        self._read_config(config_path, compose_bin)
-
-    def _read_env(self, dotenv_path, params):
-        """
-        Read .env and merge it with explicitly passed parameters.
-        """
-        self.dotenv = dotenv_values(str(dotenv_path))
-        if params is None:
-            self.params = {}
-        else:
-            self.params = {k: v for k, v in params.items() if k in self.dotenv}
-
-        # forward the process' environment variables
-        self.env = os.environ.copy()
-        # set the defaults from the dotenv files
-        self.env.update(self.dotenv)
-        # override the defaults passed as parameters
-        self.env.update(self.params)
-
-        # translate docker's architecture notation to a more widely used one
-        arch = self.env.get('ARCH', 'amd64')
-        self.env['ARCH_ALIAS'] = _arch_alias_mapping.get(arch, arch)
-        self.env['ARCH_SHORT'] = _arch_short_mapping.get(arch, arch)
-
-    def _read_config(self, config_path, compose_bin):
-        """
-        Validate and read the docker-compose.yml
-        """
-        yaml = YAML()
-        with config_path.open() as fp:
-            config = yaml.load(fp)
-
-        services = config['services'].keys()
-        self.hierarchy = dict(flatten(config.get('x-hierarchy', {})))
-        self.with_gpus = config.get('x-with-gpus', [])
-        nodes = self.hierarchy.keys()
-        errors = []
-
-        for name in self.with_gpus:
-            if name not in services:
-                errors.append(
-                    'Service `{}` defined in `x-with-gpus` bot not in '
-                    '`services`'.format(name)
-                )
-        for name in nodes - services:
-            errors.append(
-                'Service `{}` is defined in `x-hierarchy` bot not in '
-                '`services`'.format(name)
-            )
-        for name in services - nodes:
-            errors.append(
-                'Service `{}` is defined in `services` but not in '
-                '`x-hierarchy`'.format(name)
-            )
-
-        # trigger docker-compose's own validation
-        compose = Command('docker-compose')
-        args = ['--file', str(config_path), 'config']
-        result = compose.run(*args, env=self.env, check=False,
-                             stderr=subprocess.PIPE, stdout=subprocess.PIPE)
-
-        if result.returncode != 0:
-            # strip the intro line of docker-compose errors
-            errors += result.stderr.decode().splitlines()
-
-        if errors:
-            msg = '\n'.join([' - {}'.format(msg) for msg in errors])
-            raise ValueError(
-                'Found errors with docker-compose:\n{}'.format(msg)
-            )
-
-        rendered_config = StringIO(result.stdout.decode())
-        self.path = config_path
-        self.config = yaml.load(rendered_config)
-
-    def get(self, service_name):
-        try:
-            service = self.config['services'][service_name]
-        except KeyError:
-            raise UndefinedImage(service_name)
-        service['name'] = service_name
-        service['need_gpu'] = service_name in self.with_gpus
-        service['ancestors'] = self.hierarchy[service_name]
-        return service
-
-    def __getitem__(self, service_name):
-        return self.get(service_name)
-
-
-class Docker(Command):
-
-    def __init__(self, docker_bin=None):
-        self.bin = default_bin(docker_bin, "docker")
-
-
-class DockerCompose(Command):
-
-    def __init__(self, config_path, dotenv_path=None, compose_bin=None,
-                 params=None):
-        compose_bin = default_bin(compose_bin, 'docker-compose')
-        self.config = ComposeConfig(config_path, dotenv_path, compose_bin,
-                                    params)
-        self.bin = compose_bin
-        self.pull_memory = set()
-
-    def clear_pull_memory(self):
-        self.pull_memory = set()
-
-    def _execute_compose(self, *args, **kwargs):
-        # execute as a docker compose command
-        try:
-            result = super().run('--file', str(self.config.path), *args,
-                                 env=self.config.env, **kwargs)
-            result.check_returncode()
-        except subprocess.CalledProcessError as e:
-            def formatdict(d, template):
-                return '\n'.join(
-                    template.format(k, v) for k, v in sorted(d.items())
-                )
-            msg = (
-                "`{cmd}` exited with a non-zero exit code {code}, see the "
-                "process log above.\n\nThe docker-compose command was "
-                "invoked with the following parameters:\n\nDefaults defined "
-                "in .env:\n{dotenv}\n\nArchery was called with:\n{params}"
-            )
-            raise RuntimeError(
-                msg.format(
-                    cmd=' '.join(e.cmd),
-                    code=e.returncode,
-                    dotenv=formatdict(self.config.dotenv, template='  {}: {}'),
-                    params=formatdict(
-                        self.config.params, template='  export {}={}'
-                    )
-                )
-            )
-
-    def _execute_docker(self, *args, **kwargs):
-        # execute as a plain docker cli command
-        try:
-            result = Docker().run(*args, **kwargs)
-            result.check_returncode()
-        except subprocess.CalledProcessError as e:
-            raise RuntimeError(
-                "{} exited with non-zero exit code {}".format(
-                    ' '.join(e.cmd), e.returncode
-                )
-            )
-
-    def pull(self, service_name, pull_leaf=True, using_docker=False):
-        def _pull(service):
-            args = ['pull']
-            if service['image'] in self.pull_memory:
-                return
-
-            if using_docker:
-                try:
-                    self._execute_docker(*args, service['image'])
-                except Exception as e:
-                    # better --ignore-pull-failures handling
-                    print(e)
-            else:
-                args.append('--ignore-pull-failures')
-                self._execute_compose(*args, service['name'])
-
-            self.pull_memory.add(service['image'])
-
-        service = self.config.get(service_name)
-        for ancestor in service['ancestors']:
-            _pull(self.config.get(ancestor))
-        if pull_leaf:
-            _pull(service)
-
-    def build(self, service_name, use_cache=True, use_leaf_cache=True,
-              using_docker=False, using_buildx=False):
-        def _build(service, use_cache):
-            if 'build' not in service:
-                # nothing to do
-                return
-
-            args = []
-            cache_from = list(service.get('build', {}).get('cache_from', []))
-            if use_cache:
-                for image in cache_from:
-                    if image not in self.pull_memory:
-                        try:
-                            self._execute_docker('pull', image)
-                        except Exception as e:
-                            print(e)
-                        finally:
-                            self.pull_memory.add(image)
-            else:
-                args.append('--no-cache')
-
-            # turn on inline build cache, this is a docker buildx feature
-            # used to bundle the image build cache to the pushed image manifest
-            # so the build cache can be reused across hosts, documented at
-            # https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
-            if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1':
-                args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1'])
-
-            if using_buildx:
-                for k, v in service['build'].get('args', {}).items():
-                    args.extend(['--build-arg', '{}={}'.format(k, v)])
-
-                if use_cache:
-                    cache_ref = '{}-cache'.format(service['image'])
-                    cache_from = 'type=registry,ref={}'.format(cache_ref)
-                    cache_to = (
-                        'type=registry,ref={},mode=max'.format(cache_ref)
-                    )
-                    args.extend([
-                        '--cache-from', cache_from,
-                        '--cache-to', cache_to,
-                    ])
-
-                args.extend([
-                    '--output', 'type=docker',
-                    '-f', service['build']['dockerfile'],
-                    '-t', service['image'],
-                    service['build'].get('context', '.')
-                ])
-                self._execute_docker("buildx", "build", *args)
-            elif using_docker:
-                # better for caching
-                for k, v in service['build'].get('args', {}).items():
-                    args.extend(['--build-arg', '{}={}'.format(k, v)])
-                for img in cache_from:
-                    args.append('--cache-from="{}"'.format(img))
-                args.extend([
-                    '-f', service['build']['dockerfile'],
-                    '-t', service['image'],
-                    service['build'].get('context', '.')
-                ])
-                self._execute_docker("build", *args)
-            else:
-                self._execute_compose("build", *args, service['name'])
-
-        service = self.config.get(service_name)
-        # build ancestor services
-        for ancestor in service['ancestors']:
-            _build(self.config.get(ancestor), use_cache=use_cache)
-        # build the leaf/target service
-        _build(service, use_cache=use_cache and use_leaf_cache)
-
-    def run(self, service_name, command=None, *, env=None, volumes=None,
-            user=None, using_docker=False):
-        service = self.config.get(service_name)
-
-        args = []
-        if user is not None:
-            args.extend(['-u', user])
-
-        if env is not None:
-            for k, v in env.items():
-                args.extend(['-e', '{}={}'.format(k, v)])
-
-        if volumes is not None:
-            for volume in volumes:
-                args.extend(['--volume', volume])
-
-        if using_docker or service['need_gpu']:
-            # use gpus, requires docker>=19.03
-            if service['need_gpu']:
-                args.extend(['--gpus', 'all'])
-
-            if service.get('shm_size'):
-                args.extend(['--shm-size', service['shm_size']])
-
-            # append env variables from the compose conf
-            for k, v in service.get('environment', {}).items():
-                args.extend(['-e', '{}={}'.format(k, v)])
-
-            # append volumes from the compose conf
-            for v in service.get('volumes', []):
-                if not isinstance(v, str):
-                    # if not the compact string volume definition
-                    v = "{}:{}".format(v['source'], v['target'])
-                args.extend(['-v', v])
-
-            # infer whether an interactive shell is desired or not
-            if command in ['cmd.exe', 'bash', 'sh', 'powershell']:
-                args.append('-it')
-
-            # get the actual docker image name instead of the compose service
-            # name which we refer as image in general
-            args.append(service['image'])
-
-            # add command from compose if it wasn't overridden
-            if command is not None:
-                args.append(command)
-            else:
-                # replace whitespaces from the preformatted compose command
-                cmd = _sanitize_command(service.get('command', ''))
-                if cmd:
-                    args.append(cmd)
-
-            # execute as a plain docker cli command
-            self._execute_docker('run', '--rm', *args)
-        else:
-            # execute as a docker-compose command
-            args.append(service_name)
-            if command is not None:
-                args.append(command)
-            self._execute_compose('run', '--rm', *args)
-
-    def push(self, service_name, user=None, password=None, using_docker=False):
-        def _push(service):
-            if using_docker:
-                return self._execute_docker('push', service['image'])
-            else:
-                return self._execute_compose('push', service['name'])
-
-        if user is not None:
-            try:
-                # TODO(kszucs): have an option for a prompt
-                self._execute_docker('login', '-u', user, '-p', password)
-            except subprocess.CalledProcessError:
-                # hide credentials
-                msg = ('Failed to push `{}`, check the passed credentials'
-                       .format(service_name))
-                raise RuntimeError(msg) from None
-
-        service = self.config.get(service_name)
-        for ancestor in service['ancestors']:
-            _push(self.config.get(ancestor))
-        _push(service)
-
-    def images(self):
-        return sorted(self.config.hierarchy.keys())
diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py
index aaf16bdfa6e..dfdf3f4778a 100644
--- a/dev/archery/archery/docker/core.py
+++ b/dev/archery/archery/docker/core.py
@@ -48,6 +48,15 @@ def _sanitize_command(cmd):
     return re.sub(r"\s+", " ", cmd)
 
 
+_arch_short_mapping = {
+    'arm64v8': 'arm64',
+}
+_arch_alias_mapping = {
+    'amd64': 'x86_64',
+    'arm64v8': 'aarch64',
+}
+
+
 class UndefinedImage(Exception):
     pass
 
@@ -82,18 +91,8 @@ def _read_env(self, dotenv_path, params):
 
         # translate docker's architecture notation to a more widely used one
         arch = self.env.get('ARCH', 'amd64')
-        arch_aliases = {
-            'amd64': 'x86_64',
-            'arm64v8': 'aarch64',
-            's390x': 's390x'
-        }
-        arch_short_aliases = {
-            'amd64': 'x64',
-            'arm64v8': 'arm64',
-            's390x': 's390x'
-        }
-        self.env['ARCH_ALIAS'] = arch_aliases.get(arch, arch)
-        self.env['ARCH_SHORT_ALIAS'] = arch_short_aliases.get(arch, arch)
+        self.env['ARCH_ALIAS'] = _arch_alias_mapping.get(arch, arch)
+        self.env['ARCH_SHORT'] = _arch_short_mapping.get(arch, arch)
 
     def _read_config(self, config_path, compose_bin):
         """

From 3560ce867dc458b21973ef1c3461c6515e0eb87c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 04:15:45 +0100
Subject: [PATCH 24/33] Ensure that universal2-osx-static-release directory
 exists on M1

---
 dev/tasks/python-wheels/github.osx.arm64.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 543e22e4c3e..d9fe5dad831 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -76,6 +76,7 @@ jobs:
       - name: Create universal binaries for OpenSSL
         shell: bash
         run: |
+          mkdir -p $VCPKG_ROOT/installed/universal2-osx-static-release
           for arch in arm64 amd64; do
             arch -arm64 $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
           done

From c43db1d7fb6b589ecff7ba98da65ddde09ff0fa0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 04:26:57 +0100
Subject: [PATCH 25/33] Missing /lib

---
 dev/tasks/python-wheels/github.osx.arm64.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index d9fe5dad831..1a545b41518 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -76,7 +76,7 @@ jobs:
       - name: Create universal binaries for OpenSSL
         shell: bash
         run: |
-          mkdir -p $VCPKG_ROOT/installed/universal2-osx-static-release
+          mkdir -p $VCPKG_ROOT/installed/universal2-osx-static-release/lib
           for arch in arm64 amd64; do
             arch -arm64 $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
           done

From 4efa1bd6bff8114512456d8fc829f80dde4e451a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 04:52:11 +0100
Subject: [PATCH 26/33] No need to escape semicolon for defining
 VCPKG_OSX_ARCHITECTURES anymore

---
 ci/scripts/python_wheel_macos_build.sh       | 2 ++
 ci/scripts/python_wheel_manylinux_build.sh   | 2 ++
 ci/vcpkg/universal2-osx-static-debug.cmake   | 2 +-
 ci/vcpkg/universal2-osx-static-release.cmake | 2 +-
 4 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh
index 1a52a2ad52b..4689a251aa0 100755
--- a/ci/scripts/python_wheel_macos_build.sh
+++ b/ci/scripts/python_wheel_macos_build.sh
@@ -64,6 +64,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_DATASET:=ON}
 : ${ARROW_FLIGHT:=ON}
 : ${ARROW_GANDIVA:=OFF}
+: ${ARROW_GCS:=OFF}
 : ${ARROW_HDFS:=ON}
 : ${ARROW_JEMALLOC:=ON}
 : ${ARROW_MIMALLOC:=ON}
@@ -99,6 +100,7 @@ cmake \
     -DARROW_DEPENDENCY_USE_SHARED=OFF \
     -DARROW_FLIGHT=${ARROW_FLIGHT} \
     -DARROW_GANDIVA=${ARROW_GANDIVA} \
+    -DARROW_GCS=${ARROW_GCS} \
     -DARROW_HDFS=${ARROW_HDFS} \
     -DARROW_JEMALLOC=${ARROW_JEMALLOC} \
     -DARROW_MIMALLOC=${ARROW_MIMALLOC} \
diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
index 938ada84952..6b467ffc046 100755
--- a/ci/scripts/python_wheel_manylinux_build.sh
+++ b/ci/scripts/python_wheel_manylinux_build.sh
@@ -51,6 +51,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_DATASET:=ON}
 : ${ARROW_FLIGHT:=ON}
 : ${ARROW_GANDIVA:=OFF}
+: ${ARROW_GCS:=OFF}
 : ${ARROW_HDFS:=ON}
 : ${ARROW_JEMALLOC:=ON}
 : ${ARROW_MIMALLOC:=ON}
@@ -90,6 +91,7 @@ cmake \
     -DARROW_DEPENDENCY_USE_SHARED=OFF \
     -DARROW_FLIGHT=${ARROW_FLIGHT} \
     -DARROW_GANDIVA=${ARROW_GANDIVA} \
+    -DARROW_GCS=${ARROW_GCS} \
     -DARROW_HDFS=${ARROW_HDFS} \
     -DARROW_JEMALLOC=${ARROW_JEMALLOC} \
     -DARROW_MIMALLOC=${ARROW_MIMALLOC} \
diff --git a/ci/vcpkg/universal2-osx-static-debug.cmake b/ci/vcpkg/universal2-osx-static-debug.cmake
index 706ac47a72c..29e4b0e63c5 100644
--- a/ci/vcpkg/universal2-osx-static-debug.cmake
+++ b/ci/vcpkg/universal2-osx-static-debug.cmake
@@ -20,7 +20,7 @@ set(VCPKG_CRT_LINKAGE dynamic)
 set(VCPKG_LIBRARY_LINKAGE static)
 
 set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
-set(VCPKG_OSX_ARCHITECTURES "x86_64\;arm64")
+set(VCPKG_OSX_ARCHITECTURES "x86_64;arm64")
 set(VCPKG_OSX_DEPLOYMENT_TARGET "10.13")
 
 set(VCPKG_BUILD_TYPE debug)
diff --git a/ci/vcpkg/universal2-osx-static-release.cmake b/ci/vcpkg/universal2-osx-static-release.cmake
index 8670690171e..8111169fab2 100644
--- a/ci/vcpkg/universal2-osx-static-release.cmake
+++ b/ci/vcpkg/universal2-osx-static-release.cmake
@@ -20,7 +20,7 @@ set(VCPKG_CRT_LINKAGE dynamic)
 set(VCPKG_LIBRARY_LINKAGE static)
 
 set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
-set(VCPKG_OSX_ARCHITECTURES "x86_64\;arm64")
+set(VCPKG_OSX_ARCHITECTURES "x86_64;arm64")
 set(VCPKG_OSX_DEPLOYMENT_TARGET "10.13")
 
 set(VCPKG_BUILD_TYPE release)

From 86daaccfdffe7d523e26ddbe4051495f2e74f561 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 05:06:57 +0100
Subject: [PATCH 27/33] Try to run vcpkg under rosetta

---
 dev/tasks/python-wheels/github.osx.arm64.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 1a545b41518..8a7c42bcf8e 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -65,7 +65,7 @@ jobs:
         shell: bash
         env:
           MACOSX_DEPLOYMENT_TARGET: "11.0"
-        run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
+        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
       {% if arch == "universal2" %}
       # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is
@@ -78,7 +78,7 @@ jobs:
         run: |
           mkdir -p $VCPKG_ROOT/installed/universal2-osx-static-release/lib
           for arch in arm64 amd64; do
-            arch -arm64 $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
+            $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
           done
           for lib in libcrypto libssl; do
             lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \
@@ -88,12 +88,12 @@ jobs:
       {% else %}
       - name: Install OpenSSL
         shell: bash
-        run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl:arm64-osx-static-release
+        run: $VCPKG_ROOT/vcpkg install openssl:arm64-osx-static-release
       {% endif %}
 
       - name: Install Packages
         run: |
-          arch -arm64 $VCPKG_ROOT/vcpkg install \
+          $VCPKG_ROOT/vcpkg install \
             aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
             boost-filesystem \
             brotli \

From c9bfce964bb448e0994a1cd28e8dda2c7f437d99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 05:20:08 +0100
Subject: [PATCH 28/33] Only install absl under rosetta

---
 dev/tasks/python-wheels/github.osx.arm64.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 8a7c42bcf8e..f196939dfb0 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -78,7 +78,7 @@ jobs:
         run: |
           mkdir -p $VCPKG_ROOT/installed/universal2-osx-static-release/lib
           for arch in arm64 amd64; do
-            $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
+            arch -arm64 $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
           done
           for lib in libcrypto libssl; do
             lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \
@@ -93,7 +93,11 @@ jobs:
 
       - name: Install Packages
         run: |
-          $VCPKG_ROOT/vcpkg install \
+          arch -x86_64 $VCPKG_ROOT/vcpkg install abseil
+
+      - name: Install Packages
+        run: |
+          arch -arm64 $VCPKG_ROOT/vcpkg install \
             aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
             boost-filesystem \
             brotli \

From f8c7af416d679d37478c08d7c1491039d422440f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 05:25:46 +0100
Subject: [PATCH 29/33] Install arm vcpkg

---
 dev/tasks/python-wheels/github.osx.arm64.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index f196939dfb0..801f0eb22b4 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -65,7 +65,7 @@ jobs:
         shell: bash
         env:
           MACOSX_DEPLOYMENT_TARGET: "11.0"
-        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
+        run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
       {% if arch == "universal2" %}
       # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is
@@ -88,7 +88,7 @@ jobs:
       {% else %}
       - name: Install OpenSSL
         shell: bash
-        run: $VCPKG_ROOT/vcpkg install openssl:arm64-osx-static-release
+        run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl:arm64-osx-static-release
       {% endif %}
 
       - name: Install Packages

From 066279319f88dfa6a296ab59ad867e0f4156056a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 11:29:50 +0100
Subject: [PATCH 30/33] Upgrade windows base image; manually create universal2
 binary for s2n

---
 ci/docker/python-wheel-windows-vs2017.dockerfile | 2 +-
 dev/tasks/python-wheels/github.osx.arm64.yml     | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
index f7e581daf5b..e7ff011629b 100644
--- a/ci/docker/python-wheel-windows-vs2017.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -17,7 +17,7 @@
 
 # based on mcr.microsoft.com/windows/servercore:ltsc2019
 # contains choco and vs2017 preinstalled
-FROM abrarov/msvc-2017:2.10.0
+FROM abrarov/msvc-2017:2.12.1
 
 # Install CMake and Ninja
 RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' && \
diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index 801f0eb22b4..edcca0610cf 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -73,14 +73,15 @@ jobs:
       # In order to prevent link time warnings (which may turn out to be errors)
       # we compile OpenSSL separately for the two architectures and merge the
       # binaries into universal2 ones using `lipo`.
-      - name: Create universal binaries for OpenSSL
+      - name: Create universal binaries for OpenSSL and s2n
         shell: bash
         run: |
           mkdir -p $VCPKG_ROOT/installed/universal2-osx-static-release/lib
           for arch in arm64 amd64; do
             arch -arm64 $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
+            arch -arm64 $VCPKG_ROOT/vcpkg install s2n:${arch}-osx-static-release
           done
-          for lib in libcrypto libssl; do
+          for lib in libcrypto libssl s2n; do
             lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \
                          $VCPKG_ROOT/installed/amd64-osx-static-release/lib/${lib}.a \
                  -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a

From 283d7a86a61361eeb17df13e1958fe275d142549 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Fri, 26 Nov 2021 11:58:29 +0100
Subject: [PATCH 31/33] Overwrite universal2 openssl package manually

---
 dev/tasks/python-wheels/github.osx.arm64.yml | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml
index edcca0610cf..60a3c61e5b2 100644
--- a/dev/tasks/python-wheels/github.osx.arm64.yml
+++ b/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -67,35 +67,30 @@ jobs:
           MACOSX_DEPLOYMENT_TARGET: "11.0"
         run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
 
+      - name: Install OpenSSL
+        shell: bash
+        run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl:arm64-osx-static-release
+
       {% if arch == "universal2" %}
       # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is
       # unable to propagate the list of architectures from VCPKG_OSX_ARCHIETCTURES.
       # In order to prevent link time warnings (which may turn out to be errors)
       # we compile OpenSSL separately for the two architectures and merge the
       # binaries into universal2 ones using `lipo`.
-      - name: Create universal binaries for OpenSSL and s2n
+      - name: Create universal binaries for OpenSSL
         shell: bash
         run: |
           mkdir -p $VCPKG_ROOT/installed/universal2-osx-static-release/lib
           for arch in arm64 amd64; do
             arch -arm64 $VCPKG_ROOT/vcpkg install openssl:${arch}-osx-static-release
-            arch -arm64 $VCPKG_ROOT/vcpkg install s2n:${arch}-osx-static-release
           done
-          for lib in libcrypto libssl s2n; do
+          for lib in libcrypto libssl; do
             lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \
                          $VCPKG_ROOT/installed/amd64-osx-static-release/lib/${lib}.a \
                  -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a
           done
-      {% else %}
-      - name: Install OpenSSL
-        shell: bash
-        run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl:arm64-osx-static-release
       {% endif %}
 
-      - name: Install Packages
-        run: |
-          arch -x86_64 $VCPKG_ROOT/vcpkg install abseil
-
       - name: Install Packages
         run: |
           arch -arm64 $VCPKG_ROOT/vcpkg install \

From d221c48da25afbc2e767198e648c34fbcee576fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Mon, 29 Nov 2021 11:11:28 +0100
Subject: [PATCH 32/33] Remove unconditional image push

---
 dev/tasks/python-wheels/github.linux.amd64.yml | 2 +-
 dev/tasks/python-wheels/travis.linux.arm64.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/dev/tasks/python-wheels/github.linux.amd64.yml b/dev/tasks/python-wheels/github.linux.amd64.yml
index 7e079172a0a..dc2386482f1 100644
--- a/dev/tasks/python-wheels/github.linux.amd64.yml
+++ b/dev/tasks/python-wheels/github.linux.amd64.yml
@@ -47,7 +47,7 @@ jobs:
       {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
       {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
 
-      {% if True or arrow.branch == 'master' %}
+      {% if arrow.branch == 'master' %}
       - name: Push Docker Image
         shell: bash
         run: |
diff --git a/dev/tasks/python-wheels/travis.linux.arm64.yml b/dev/tasks/python-wheels/travis.linux.arm64.yml
index 7ee2f164dfd..d32d89d8301 100644
--- a/dev/tasks/python-wheels/travis.linux.arm64.yml
+++ b/dev/tasks/python-wheels/travis.linux.arm64.yml
@@ -66,7 +66,7 @@ after_success:
   {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }}
   {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }}
 
-  {% if True or arrow.branch == 'master' %}
+  {% if arrow.branch == 'master' %}
   # Push the docker image to dockerhub
   - archery docker push python-wheel-manylinux-{{ manylinux_version }}
   - archery docker push python-wheel-manylinux-test-unittests

From e31380b3fcfce9d01d95ca6ad2eae53aaa2e4cfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= <szucs.krisztian@gmail.com>
Date: Tue, 30 Nov 2021 14:29:35 +0100
Subject: [PATCH 33/33] Use the number of available cores to build glibc

---
 ci/scripts/install_glibc.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/install_glibc.sh b/ci/scripts/install_glibc.sh
index aecd2893a78..d1a49818178 100755
--- a/ci/scripts/install_glibc.sh
+++ b/ci/scripts/install_glibc.sh
@@ -35,7 +35,7 @@ wget -q ${url} -O - | tar -xzf - --directory /tmp/glibc --strip-components=1
 mkdir /tmp/glibc/build
 pushd /tmp/glibc/build
 ../configure --prefix=${prefix}
-make -j4
+make -j$(nproc)
 make install
 popd