From 92c6029f09136fbc10b97b5a37896e2ac463818e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Wed, 30 Jun 2021 23:38:02 +0200 Subject: [PATCH 1/7] install vcpkg script --- ci/scripts/install_vcpkg.sh | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 ci/scripts/install_vcpkg.sh diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh new file mode 100755 index 00000000000..d061711e62b --- /dev/null +++ b/ci/scripts/install_vcpkg.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +vcpkg_version=$1 +vcpkg_destination=$2 +vcpkg_patch=$(realpath $(dirname "${0}")/../vcpkg/ports.patch) + +git clone https://github.com/microsoft/vcpkg ${vcpkg_destination} + +pushd ${vcpkg_destination} + +git checkout ${vcpkg_version} +./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics + +if ! git apply --reverse --check --ignore-whitespace ${vcpkg_patch}; then + git apply --ignore-whitespace ${vcpkg_patch} + echo "Patch successfully applied!" +fi + +popd From d04c29178e96eb18d58c3e10d6c0849292fb15ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 1 Jul 2021 01:28:33 +0200 Subject: [PATCH 2/7] quicker clone --- ci/scripts/install_vcpkg.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh index d061711e62b..e45e267e476 100755 --- a/ci/scripts/install_vcpkg.sh +++ b/ci/scripts/install_vcpkg.sh @@ -28,11 +28,10 @@ vcpkg_version=$1 vcpkg_destination=$2 vcpkg_patch=$(realpath $(dirname "${0}")/../vcpkg/ports.patch) -git clone https://github.com/microsoft/vcpkg ${vcpkg_destination} +git clone --depth 1 --branch ${vcpkg_version} https://github.com/microsoft/vcpkg ${vcpkg_destination} pushd ${vcpkg_destination} -git checkout ${vcpkg_version} ./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics if ! git apply --reverse --check --ignore-whitespace ${vcpkg_patch}; then @@ -41,3 +40,6 @@ if ! git apply --reverse --check --ignore-whitespace ${vcpkg_patch}; then fi popd + +# ln -s ${vcpkg_destination}/vcpkg /usr/bin/vcpkg +# buildtrees/downloads/installed/packages From 752833a66778684a4a6c7ea2206f88bda180165d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 1 Jul 2021 03:26:57 +0200 Subject: [PATCH 3/7] update macos config --- dev/tasks/python-wheels/github.osx.yml | 46 +++++++++----------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index 84b094b84d1..7e47ace79ff 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -30,7 +30,6 @@ env: VCPKG_DEFAULT_TRIPLET: x64-osx-static-release VCPKG_FEATURE_FLAGS: "-manifests" VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} - ARROW_VCPKG_PATCH_FILE: {{ "${{ github.workspace }}/arrow/ci/vcpkg/ports.patch" }} jobs: build: @@ -40,36 +39,23 @@ jobs: {{ macros.github_checkout_arrow()|indent }} - name: Install System Dependencies - run: brew install bison ninja - - # Restore from cache the previously built ports. - # If cache-miss, download and build vcpkg (aka "bootstrap vcpkg"). - - name: Restore from Cache and Install Vcpkg - # Download and build vcpkg, without installing any port. - # If content is cached already, it is a no-op. - uses: kszucs/run-vcpkg@main + run: brew install bison coreutils ninja cmake + + - uses: actions/cache@v2 + id: vcpkg-cache with: - # Required to prevent cache eviction on crossbow's main branch - # where we build pre-build the vcpkg packages - setupOnly: true - doNotSaveCache: true - appendedCacheKey: "-macos-{{ macos_deployment_target }}-2021-06-25" - vcpkgDirectory: {{ "${{ github.workspace }}/vcpkg" }} - vcpkgGitCommitId: "2021.04.30" - - - name: Patch Vcpkg Ports + path: vcpkg + key: vcpkg-${{ matrix.macos_version }}-${{ env.VCPKG_VERSION }}-${{ hashFiles('arrow/ci/vcpkg/**') }}-4 + + - name: Install Vcpkg + # if: steps.vcpkg-cache.outputs.cache-hit != 'true' + shell: bash run: | - set -ex - cd $VCPKG_ROOT - if ! git apply --reverse --check --ignore-whitespace ${ARROW_VCPKG_PATCH_FILE}; then - git apply --ignore-whitespace ${ARROW_VCPKG_PATCH_FILE} - echo "Patch successfully applied!" - fi - - # Now that vcpkg is installed, it is being used to run with the desired arguments. - - name: Install Vcpkg Dependencies + [ -d "vcpkg" ] || arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION vcpkg + + - name: Install Packages run: | - $VCPKG_ROOT/vcpkg install \ + vcpkg/vcpkg install \ abseil \ boost-filesystem \ brotli \ @@ -94,9 +80,7 @@ jobs: {% if arrow_s3 == "ON" %} - name: Install AWS SDK C++ - run: | - $VCPKG_ROOT/vcpkg install \ - aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] + run: vcpkg/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] {% endif %} - name: Setup Multibuild From 6e8d02c582e9f4e8582f76e81b15c11285af48a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 1 Jul 2021 03:29:57 +0200 Subject: [PATCH 4/7] update macos config --- dev/tasks/python-wheels/github.osx.yml | 2 +- dev/tasks/tasks.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index 7e47ace79ff..ef96d3d091d 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -45,7 +45,7 @@ jobs: id: vcpkg-cache with: path: vcpkg - key: vcpkg-${{ matrix.macos_version }}-${{ env.VCPKG_VERSION }}-${{ hashFiles('arrow/ci/vcpkg/**') }}-4 + key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/**') }}" }}-4 - name: Install Vcpkg # if: steps.vcpkg-cache.outputs.cache-hit != 'true' diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 34307bb2583..963e7dadc26 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -359,6 +359,7 @@ tasks: ci: github template: python-wheels/github.osx.yml params: + vcpkg_version: "2021.04.30" python_version: {{ python_version }} macos_deployment_target: {{ macos_version }} arrow_s3: {{ arrow_s3 }} From 08af0007ec9d3423a08b867a3fbb754e85736b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 1 Jul 2021 03:49:12 +0200 Subject: [PATCH 5/7] VCPKG_ROOT --- dev/tasks/python-wheels/github.osx.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index ef96d3d091d..cbcd56d92cb 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -30,6 +30,7 @@ env: VCPKG_DEFAULT_TRIPLET: x64-osx-static-release VCPKG_FEATURE_FLAGS: "-manifests" VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} + VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} jobs: build: @@ -51,11 +52,11 @@ jobs: # if: steps.vcpkg-cache.outputs.cache-hit != 'true' shell: bash run: | - [ -d "vcpkg" ] || arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION vcpkg + [ -d "vcpkg" ] || arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT - name: Install Packages run: | - vcpkg/vcpkg install \ + $VCPKG_ROOT/vcpkg install \ abseil \ boost-filesystem \ brotli \ @@ -80,7 +81,7 @@ jobs: {% if arrow_s3 == "ON" %} - name: Install AWS SDK C++ - run: vcpkg/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] + run: $VCPKG_ROOT/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] {% endif %} - name: Setup Multibuild From b140be3293fadacedc1b83cfa62aa86b8df4f788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Thu, 1 Jul 2021 16:12:04 +0200 Subject: [PATCH 6/7] Cleanup --- ci/scripts/install_vcpkg.sh | 3 --- dev/tasks/python-wheels/github.osx.yml | 7 +++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh index e45e267e476..ea6cdec105e 100755 --- a/ci/scripts/install_vcpkg.sh +++ b/ci/scripts/install_vcpkg.sh @@ -40,6 +40,3 @@ if ! git apply --reverse --check --ignore-whitespace ${vcpkg_patch}; then fi popd - -# ln -s ${vcpkg_destination}/vcpkg /usr/bin/vcpkg -# buildtrees/downloads/installed/packages diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.yml index cbcd56d92cb..a2e5c0af21b 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -46,13 +46,12 @@ jobs: id: vcpkg-cache with: path: vcpkg - key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/**') }}" }}-4 + key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/**') }}" }} - name: Install Vcpkg - # if: steps.vcpkg-cache.outputs.cache-hit != 'true' + if: steps.vcpkg-cache.outputs.cache-hit != 'true' shell: bash - run: | - [ -d "vcpkg" ] || arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT + run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT - name: Install Packages run: | From 2c337475c44794560c15017ac047ce1bcbea8af8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Fri, 2 Jul 2021 12:21:32 +0200 Subject: [PATCH 7/7] Always apply the patch --- ci/scripts/install_vcpkg.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh index ea6cdec105e..fe99a7fea2f 100755 --- a/ci/scripts/install_vcpkg.sh +++ b/ci/scripts/install_vcpkg.sh @@ -33,10 +33,7 @@ git clone --depth 1 --branch ${vcpkg_version} https://github.com/microsoft/vcpkg pushd ${vcpkg_destination} ./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics - -if ! git apply --reverse --check --ignore-whitespace ${vcpkg_patch}; then - git apply --ignore-whitespace ${vcpkg_patch} - echo "Patch successfully applied!" -fi +git apply --ignore-whitespace ${vcpkg_patch} +echo "Patch successfully applied!" popd