From 2a9be02aca417370ec6344b26560113d1fe93e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 28 Jul 2023 11:32:28 +0200 Subject: [PATCH 1/6] GH-36752: [Python] Remove AWS SDK bundling when building wheels --- ci/docker/python-wheel-manylinux.dockerfile | 3 ++- ci/docker/python-wheel-windows-vs2017.dockerfile | 3 ++- ci/scripts/java_jni_macos_build.sh | 1 - ci/scripts/python_wheel_manylinux_build.sh | 4 ---- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile index ed329ecdef6..0f7779c8785 100644 --- a/ci/docker/python-wheel-manylinux.dockerfile +++ b/ci/docker/python-wheel-manylinux.dockerfile @@ -84,7 +84,8 @@ RUN vcpkg install \ --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + --x-feature=s3 # Configure Python for applications running in the bash shell of this Dockerfile ARG python=3.8 diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 01152dae232..531c4e678fa 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -66,7 +66,8 @@ RUN vcpkg install \ --x-feature=flight \ --x-feature=gcs \ --x-feature=json \ - --x-feature=parquet + --x-feature=parquet \ + --x-feature=s3 # Remove previous installations of python from the base image # NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7 diff --git a/ci/scripts/java_jni_macos_build.sh b/ci/scripts/java_jni_macos_build.sh index 4a6f9444ec2..d66c39a37c5 100755 --- a/ci/scripts/java_jni_macos_build.sh +++ b/ci/scripts/java_jni_macos_build.sh @@ -81,7 +81,6 @@ cmake \ -DARROW_PARQUET=${ARROW_PARQUET} \ -DARROW_S3=${ARROW_S3} \ -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \ - -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${install_dir} \ diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh index cb5c2fbb7cc..58e42fea880 100755 --- a/ci/scripts/python_wheel_manylinux_build.sh +++ b/ci/scripts/python_wheel_manylinux_build.sh @@ -85,9 +85,6 @@ fi mkdir /tmp/arrow-build pushd /tmp/arrow-build -# ARROW-17501: We can remove -DAWSSDK_SOURCE=BUNDLED once -# https://github.com/aws/aws-sdk-cpp/issues/1809 is fixed and vcpkg -# ships the fix. cmake \ -DARROW_ACERO=${ARROW_ACERO} \ -DARROW_BUILD_SHARED=ON \ @@ -120,7 +117,6 @@ cmake \ -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \ -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \ -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \ - -DAWSSDK_SOURCE=BUNDLED \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \ From d6a45dd964ca55da7c0b2d72f615ae999203874c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 28 Jul 2023 16:37:59 +0200 Subject: [PATCH 2/6] Temporary increase timeout to allow vcpkg to build --- .github/workflows/java_jni.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 2787ac1fc71..12fd10acf4c 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -51,7 +51,7 @@ jobs: name: AMD64 manylinux2014 Java JNI runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 90 + timeout-minutes: 150 steps: - name: Checkout Arrow uses: actions/checkout@v3 From 241e89c514189e1ac1c9850006d5077fd026c5d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 28 Jul 2023 19:38:31 +0200 Subject: [PATCH 3/6] Try to give the build an extremely amount of time --- .github/workflows/java_jni.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index 12fd10acf4c..c613368503e 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -51,7 +51,7 @@ jobs: name: AMD64 manylinux2014 Java JNI runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 150 + timeout-minutes: 300 steps: - name: Checkout Arrow uses: actions/checkout@v3 From 1fb7e2ca9eccc4f73c0600f415b4e83329fc143f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Tue, 1 Aug 2023 23:47:35 +0200 Subject: [PATCH 4/6] Remove AWSSDK_SOURCE=BUNDLED from Windows wheels --- ci/scripts/python_wheel_windows_build.bat | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index ee879c7050d..ffb43b3481e 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -49,9 +49,6 @@ set ARROW_WITH_LZ4=ON set ARROW_WITH_SNAPPY=ON set ARROW_WITH_ZLIB=ON set ARROW_WITH_ZSTD=ON -@rem Workaround for https://github.com/aws/aws-sdk-cpp/issues/1809 . -@rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++. -set AWSSDK_SOURCE=BUNDLED set CMAKE_UNITY_BUILD=ON set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 set VCPKG_ROOT=C:\vcpkg @@ -90,7 +87,6 @@ cmake ^ -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^ -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^ -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^ - -DAWSSDK_SOURCE=%AWSSDK_SOURCE% ^ -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^ -DCMAKE_CXX_COMPILER=clcache ^ -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^ From c04d057776968e428e53323992f60959cac122cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 2 Aug 2023 08:50:59 +0200 Subject: [PATCH 5/6] Update Windows wheel revision to rebuild with VCPKG S3 feature enabled --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 25b2743f654..c9cd6c8094e 100644 --- a/.env +++ b/.env @@ -98,7 +98,7 @@ VCPKG="501db0f17ef6df184fcdbfbe0f87cde2313b6ab1" # 2023.04.15 Release # ci/docker/python-wheel-windows-vs2017.dockerfile. # This is a workaround for our CI problem that "archery docker build" doesn't # use pulled built images in dev/tasks/python-wheels/github.windows.yml. -PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12 +PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2023-08-02 # Use conanio/${CONAN} for "docker-compose run --rm conan". See # https://github.com/conan-io/conan-docker-tools#readme for available From 24bb448ccf3c251e90b24f8fe568e255d75aa5b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 3 Aug 2023 12:19:58 +0200 Subject: [PATCH 6/6] Revert timeout increase to build Image --- .github/workflows/java_jni.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml index c613368503e..2787ac1fc71 100644 --- a/.github/workflows/java_jni.yml +++ b/.github/workflows/java_jni.yml @@ -51,7 +51,7 @@ jobs: name: AMD64 manylinux2014 Java JNI runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} - timeout-minutes: 300 + timeout-minutes: 90 steps: - name: Checkout Arrow uses: actions/checkout@v3