diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index 8fe5878254d..03cbcc7c98f 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -79,14 +79,12 @@ jobs: matrix: platform: - { runs_on: ["macos-latest"], arch: "x86_64"} - - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], arch: "aarch_64" } + - { runs_on: ["macos-14"], arch: "aarch_64" } env: MACOSX_DEPLOYMENT_TARGET: "10.15" steps: {{ macros.github_checkout_arrow()|indent }} - name: Set up Python - if: | - !contains(matrix.platform.runs_on, 'self-hosted') uses: actions/setup-python@v4 with: cache: 'pip' @@ -134,9 +132,7 @@ jobs: run: | set -e # make brew Java available to CMake - if [ "{{ arch }}" = "aarch_64" ]; then - export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home - fi + export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home arrow/ci/scripts/java_jni_macos_build.sh \ $GITHUB_WORKSPACE/arrow \ $GITHUB_WORKSPACE/arrow/cpp-build \ diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml deleted file mode 100644 index 380c2e42f1d..00000000000 --- a/dev/tasks/python-wheels/github.osx.arm64.yml +++ /dev/null @@ -1,128 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prerequisites on the host: -# - brew install bash bison coreutils ninja cmake -# - sudo arrow/ci/scripts/install_python.sh macos 3.9 - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -env: - ARROW_JEMALLOC: OFF - CC: "clang" - CMAKE_BUILD_TYPE: release - CMAKE_CXX_COMPILER_LAUNCHER: "ccache" - CXX: "clang++" - MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" - PYARROW_BUILD_VERBOSE: 1 - PYARROW_VERSION: "{{ arrow.no_rc_version }}" - PYTHON_VERSION: "{{ python_version }}" - PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" - VCPKG_FEATURE_FLAGS: "manifests" - VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} - VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} - -jobs: - build: - name: Build wheel for Python {{ python_version }} on macOS - runs-on: ["self-hosted", "macOS", "arm64"] - steps: - - name: Cleanup - run: rm -rf arrow vcpkg build crossbow-env build-*-env test-*-env - - {{ macros.github_checkout_arrow()|indent }} - - - name: Add Brew's Bison to PATH - run: echo "/opt/homebrew/opt/bison/bin" >> $GITHUB_PATH - - - name: Retrieve VCPKG version from arrow/.env - run: | - vcpkg_version=$(cat "arrow/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"') - echo "VCPKG_VERSION=$vcpkg_version" >> $GITHUB_ENV - - - name: Install Vcpkg - env: - MACOSX_DEPLOYMENT_TARGET: "11.0" - run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION - - - name: Add Vcpkg to PATH - run: echo ${VCPKG_ROOT} >> $GITHUB_PATH - - - name: Install ARM64 Packages - env: - VCPKG_DEFAULT_TRIPLET: arm64-osx-static-release - run: | - vcpkg install \ - --clean-after-build \ - --x-install-root=${VCPKG_ROOT}/installed \ - --x-manifest-root=arrow/ci/vcpkg \ - --x-feature=azure \ - --x-feature=flight \ - --x-feature=gcs \ - --x-feature=json \ - --x-feature=parquet \ - --x-feature=s3 - - - name: Build ARM64 Wheel - env: - ARROW_SIMD_LEVEL: "DEFAULT" - VCPKG_DEFAULT_TRIPLET: arm64-osx-static-release - run: | - $PYTHON -m venv build-arm64-env - source build-arm64-env/bin/activate - pip install --upgrade pip wheel - arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build - - - uses: actions/upload-artifact@v3 - with: - name: wheel - path: arrow/python/repaired_wheels/*.whl - - - name: Test Wheel on ARM64 - shell: bash - env: - PYTEST_ADDOPTS: "-k 'not test_cancellation'" - run: | - $PYTHON -m venv test-arm64-env - source test-arm64-env/bin/activate - pip install --upgrade pip wheel - # libffi has to be installed on the m1 runner which causes issues with - # the cffi wheel. We build cffi with the flags pointing to the correct libffi location. - LDFLAGS=-L$(brew --prefix libffi)/lib CFLAGS=-I$(brew --prefix libffi)/include \ - pip install cffi --no-binary :all: - pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default - arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow - - - name: Upload artifacts - shell: bash - run: | - $PYTHON -m venv crossbow-env - source crossbow-env/bin/activate - arch -x86_64 pip install -e arrow/dev/archery[crossbow-upload] - arch -x86_64 archery crossbow \ - --queue-path $(pwd) \ - --queue-remote {{ queue_remote_url }} \ - upload-artifacts \ - --sha {{ task.branch }} \ - --tag {{ task.tag }} \ - "arrow/python/repaired_wheels/*.whl" - env: - CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }} - - {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.amd64.yml b/dev/tasks/python-wheels/github.osx.yml similarity index 80% rename from dev/tasks/python-wheels/github.osx.amd64.yml rename to dev/tasks/python-wheels/github.osx.yml index e31a681653b..11bdf031f51 100644 --- a/dev/tasks/python-wheels/github.osx.amd64.yml +++ b/dev/tasks/python-wheels/github.osx.yml @@ -19,14 +19,16 @@ {{ macros.github_header() }} env: + ARROW_JEMALLOC: "{{ arrow_jemalloc }}" CC: "clang" + CMAKE_BUILD_TYPE: release CXX: "clang++" MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" PYARROW_BUILD_VERBOSE: 1 PYARROW_VERSION: "{{ arrow.no_rc_version }}" PYTHON_VERSION: "{{ python_version }}" PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" - VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release + VCPKG_DEFAULT_TRIPLET: "{{ vcpkg_arch }}-osx-static-release" VCPKG_FEATURE_FLAGS: "manifests" VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} @@ -34,40 +36,33 @@ env: jobs: build: name: Build wheel for Python {{ python_version }} on macOS - runs-on: macos-latest + runs-on: "{{ runs_on }}" env: VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite' steps: {{ macros.github_checkout_arrow()|indent }} - name: Install System Dependencies - run: brew install bash bison coreutils ninja - - - name: Install Specific CMake version - uses: lukka/get-cmake@v3.21.2 + run: | + brew install bash bison coreutils ninja + echo "$(brew --prefix bison)/bin" >> $GITHUB_PATH - name: Retrieve VCPKG version from arrow/.env - shell: bash run: | vcpkg_version=$(cat "arrow/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"') echo "VCPKG_VERSION=$vcpkg_version" >> $GITHUB_ENV - name: Install Vcpkg - shell: bash - env: - MACOSX_DEPLOYMENT_TARGET: "10.15" run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION - name: Add Vcpkg to PATH - shell: bash run: echo ${VCPKG_ROOT} >> $GITHUB_PATH - name: Setup NuGet Credentials - shell: bash env: GITHUB_TOKEN: {{ '${{ secrets.GITHUB_TOKEN }}' }} run: | - mono `vcpkg fetch nuget | tail -n 1` \ + mono $(vcpkg fetch nuget | tail -n 1) \ sources add \ -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" \ -storepasswordincleartext \ @@ -75,7 +70,7 @@ jobs: -username "$GITHUB_REPOSITORY_OWNER" \ -password "$GITHUB_TOKEN" \ - mono `vcpkg fetch nuget | tail -n 1` \ + mono $(vcpkg fetch nuget | tail -n 1) \ setapikey "$GITHUB_TOKEN" \ -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json" @@ -93,33 +88,30 @@ jobs: --x-feature=s3 - name: Install Python {{ python_version }} - shell: bash run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }} - name: Build Wheel - shell: bash run: | $PYTHON -m venv build-env source build-env/bin/activate pip install --upgrade pip wheel - PYTHON=python arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build + PYTHON=python arrow/ci/scripts/python_wheel_macos_build.sh {{ arch }} $(pwd)/arrow $(pwd)/build - uses: actions/upload-artifact@v3 with: name: wheel path: arrow/python/repaired_wheels/*.whl - - name: Test Wheel on AMD64 - shell: bash + - name: Test Wheel env: PYTEST_ADDOPTS: "-k 'not test_cancellation'" run: | - $PYTHON -m venv test-amd64-env - source test-amd64-env/bin/activate + $PYTHON -m venv test-env + source test-env/bin/activate pip install --upgrade pip wheel - arch -x86_64 pip install -r arrow/python/requirements-wheel-test.txt - PYTHON=python arch -x86_64 arrow/ci/scripts/install_gcs_testbench.sh default - arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + arch -{{ arch }} pip install -r arrow/python/requirements-wheel-test.txt + PYTHON=python arch -{{ arch }} arrow/ci/scripts/install_gcs_testbench.sh default + arch -{{ arch }} arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 2ddfd02e731..f23da3c8773 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -66,16 +66,14 @@ jobs: fail-fast: false matrix: platform: - - { runs_on: ["self-hosted", "macos-10.13"], arch: "x86_64" } - - - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], arch: "arm64" } + - { runs_on: macos-13, arch: "x86_64" } + - { runs_on: macos-14, arch: "arm64" } openssl: ['3.0', '1.1'] steps: {{ macros.github_checkout_arrow(action_v="3")|indent }} {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - name: Install Deps - if: {{ "${{ !contains(matrix.platform.runs_on, 'macos-10.13') }}" }} run: | brew install sccache ninja brew install openssl@{{ '${{ matrix.openssl }}' }} @@ -204,50 +202,30 @@ jobs: r-packages: needs: [source, windows-cpp, macos-cpp] - name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version.r }}' }} + name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version }}' }} runs-on: {{ '${{ matrix.platform.runs_on }}' }} strategy: fail-fast: false matrix: platform: - { runs_on: 'windows-latest', name: "Windows"} - - { runs_on: macos-11 , name: "macOS Big Sur"} - - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], name: "macOS Big Sur (M1)" } + - { runs_on: macos-11, name: "macOS x86_64"} + - { runs_on: macos-14, name: "macOS arm64" } r_version: [oldrel, release] steps: - uses: r-lib/actions/setup-r@v2 - # expression marker prevents the ! being parsed as yaml tag - if: {{ "${{ !contains(matrix.platform.runs_on, 'self-hosted') }}" }} with: r-version: {{ '${{ matrix.r_version }}' }} - - name: Setup R Self-Hosted - if: contains(matrix.platform.runs_on, 'self-hosted') - run: | - # rig is a system utility that allows for switching - # between pre-installed R version on the self-hosted runners - # rig add {{ '${{ matrix.r_version }}' }} #uncomment this to install latest release/oldrel - rig default {{ '${{ matrix.r_version }}' }} - - rig system setup-user-lib {{ macros.github_setup_local_r_repo(false, true, true)|indent }} - name: Prepare Dependency Installation shell: bash run: | tar -xzf repo/src/contrib/arrow_*.tar.gz arrow/DESCRIPTION - name: Install dependencies - if: {{ "${{ !contains(matrix.platform.runs_on, 'self-hosted') }}" }} uses: r-lib/actions/setup-r-dependencies@v2 with: working-directory: 'arrow' extra-packages: cpp11 - - name: Install dependencies self-hosted - if: {{ "${{ contains(matrix.platform.runs_on, 'self-hosted') }}" }} - shell: Rscript {0} - run: | - if (!requireNamespace("devtools", quietly = TRUE)) { - install.packages("devtools") - } - devtools::install_dev_deps('./arrow') - name: Set CRAN like openssl if: contains(matrix.platform.runs_on, 'arm64') run: | diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c166ea15f37..9d604c379eb 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -393,61 +393,49 @@ tasks: - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl {% endfor %} -{############################## Wheel OSX ####################################} +{############################## Wheel macOS ####################################} {% for macos_version, macos_codename in [("10.15", "catalina")] %} {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %} wheel-macos-{{ macos_codename }}-{{ python_tag }}-amd64: ci: github - template: python-wheels/github.osx.amd64.yml + template: python-wheels/github.osx.yml params: + arch: "x86_64" + arrow_jemalloc: "ON" python_version: "{{ python_version }}" macos_deployment_target: "{{ macos_version }}" + runs_on: "macos-latest" + vcpkg_arch: "amd64" artifacts: - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl {% endfor %} -{############################## Wheel Windows ################################} - - wheel-windows-{{ python_tag }}-amd64: + wheel-macos-big-sur-{{ python_tag }}-arm64: ci: github - template: python-wheels/github.windows.yml + template: python-wheels/github.osx.yml params: + arch: "arm64" + arrow_jemalloc: "OFF" python_version: "{{ python_version }}" - artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl - -{% endfor %} - -{############################## Wheel OSX M1 #################################} - - # The python 3.8 universal2 installer has been built with macos deployment - # target 11.0, so we cannot build binaries with earlier deployment target - # otherwise distutils will raise a deployment target version mismatch error. - wheel-macos-big-sur-cp38-arm64: - ci: github - template: python-wheels/github.osx.arm64.yml - params: - arch: arm64 - arrow_simd_level: "DEFAULT" - python_version: "3.8" macos_deployment_target: "11.0" - + runs_on: "macos-14" + vcpkg_arch: "arm64" artifacts: - - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl -{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310"), ("3.11", "cp311"), ("3.12", "cp312")] %} - wheel-macos-big-sur-{{ python_tag }}-arm64: +{############################## Wheel Windows ################################} + + wheel-windows-{{ python_tag }}-amd64: ci: github - template: python-wheels/github.osx.arm64.yml + template: python-wheels/github.windows.yml params: - arch: arm64 python_version: "{{ python_version }}" - macos_deployment_target: "11.0" artifacts: - - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl + {% endfor %} {############################ Python sdist ####################################} @@ -992,7 +980,7 @@ tasks: TEST_INTEGRATION_JAVA: 0 PYTEST_ADDOPTS: "-k 'not test_cancellation'" target: {{ target }} - github_runner: ["self-hosted", "macOS", "arm64"] + github_runner: "macos-14" {% endfor %} {% for macos_version in ["11", "12"] %} diff --git a/dev/tasks/verify-rc/github.macos.arm64.yml b/dev/tasks/verify-rc/github.macos.arm64.yml index fce10925df2..925f572839e 100644 --- a/dev/tasks/verify-rc/github.macos.arm64.yml +++ b/dev/tasks/verify-rc/github.macos.arm64.yml @@ -22,7 +22,7 @@ jobs: verify: name: "Verify release candidate on macOS" - runs-on: {{ github_runner|default(["self-hosted", "macOS", "arm64"]) }} + runs-on: {{ github_runner|default("macos-14") }} {% if env is defined %} env: {% for key, value in env.items() %}