diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile index 19246a46764..ae1b0a7767c 100644 --- a/ci/docker/python-wheel-manylinux-201x.dockerfile +++ b/ci/docker/python-wheel-manylinux-201x.dockerfile @@ -58,7 +58,9 @@ RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \ ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg # Patch ports files as needed -COPY ci/vcpkg arrow/ci/vcpkg +COPY ci/vcpkg/*.patch \ + ci/vcpkg/*linux*.cmake \ + arrow/ci/vcpkg/ RUN cd /opt/vcpkg && git apply --ignore-whitespace /arrow/ci/vcpkg/ports.patch ARG build_type=release diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile index 0f66a20396e..ebf51d75d29 100644 --- a/ci/docker/python-wheel-windows-vs2017.dockerfile +++ b/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -27,14 +27,19 @@ RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=Sys RUN setx path "%path%;C:\Program Files\Git\usr\bin" # Install vcpkg +# +# Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has +# started to ship precompiled binaries for the vcpkg-tool. ARG vcpkg RUN git clone https://github.com/Microsoft/vcpkg && \ - git -C vcpkg checkout %vcpkg% && \ - vcpkg\bootstrap-vcpkg.bat -disableMetrics -win64 && \ - setx PATH "%PATH%;C:\vcpkg" + vcpkg\bootstrap-vcpkg.bat -disableMetrics && \ + setx PATH "%PATH%;C:\vcpkg" && \ + git -C vcpkg checkout %vcpkg% # Patch ports files as needed -COPY ci/vcpkg arrow/ci/vcpkg +COPY ci/vcpkg/*.patch \ + ci/vcpkg/*windows*.cmake \ + arrow/ci/vcpkg/ RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch # Configure vcpkg and install dependencies @@ -42,12 +47,12 @@ RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch # statements but bash notation in ENV statements # VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system # cmake's and ninja's versions are recent enough -COPY ci/vcpkg arrow/ci/vcpkg ARG build_type=release ENV CMAKE_BUILD_TYPE=${build_type} \ VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \ VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \ VCPKG_FEATURE_FLAGS=-manifests + RUN vcpkg install --clean-after-build \ abseil \ aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh new file mode 100755 index 00000000000..bede67216a2 --- /dev/null +++ b/ci/scripts/install_python.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +declare -A platforms +platforms=([windows]=Windows + [macos]=MacOSX + [linux]=Linux) + +declare -A versions +versions=([3.6]=3.6.8 + [3.7]=3.7.9 + [3.8]=3.8.9 + [3.9]=3.9.6) + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +elif [[ -z ${platforms[$1]} ]]; then + echo "Unexpected platform: ${1}" + exit 1 +fi + +platform=${platforms[$1]} +version=$2 +full_version=${versions[$2]} + +if [ $platform = "MacOSX" ]; then + echo "Downloading Python installer..." + if [ "$(uname -m)" = "arm64" ]; then + fname="python-${full_version}-macos11.pkg" + else + fname="python-${full_version}-macosx10.9.pkg" + fi + wget "https://www.python.org/ftp/python/${full_version}/${fname}" + + echo "Installing Python..." + installer -pkg $fname -target / + rm $fname + + echo "Installing Pip..." + python="/Library/Frameworks/Python.framework/Versions/${version}/bin/python${version}" + pip="${python} -m pip" + + $python -m ensurepip + $pip install -U pip setuptools virtualenv +else + echo "Unsupported platform: $platform" +fi diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index 93e4939af23..f9fe8f98a3e 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -19,8 +19,9 @@ set -ex -source_dir=${1} -build_dir=${2} +arch=${1} +source_dir=${2} +build_dir=${3} echo "=== (${PYTHON_VERSION}) Clear output directories and leftovers ===" # Clear output directories and leftovers @@ -31,11 +32,32 @@ rm -rf ${source_dir}/python/repaired_wheels rm -rf ${source_dir}/python/pyarrow/*.so rm -rf ${source_dir}/python/pyarrow/*.so.* -echo "=== (${PYTHON_VERSION}) Set OSX SDK and C flags ===" -# Arrow is 64-bit-only at the moment -export CFLAGS="-fPIC -arch x86_64 ${CFLAGS//-arch i386/}" -export CXXFLAGS="-fPIC -arch x86_64 ${CXXFLAGS//-arch i386} -std=c++11" -export SDKROOT="$(xcrun --show-sdk-path)" +echo "=== (${PYTHON_VERSION}) Set SDK, C++ and Wheel flags ===" +export _PYTHON_HOST_PLATFORM="macosx-${MACOSX_DEPLOYMENT_TARGET}-${arch}" +export MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET:-10.9} +export SDKROOT=${SDKROOT:-$(xcrun --sdk macosx --show-sdk-path)} + +if [ $arch = "arm64" ]; then + export CMAKE_OSX_ARCHITECTURES="arm64" +elif [ $arch = "x86_64" ]; then + export CMAKE_OSX_ARCHITECTURES="x86_64" +elif [ $arch = "universal2" ]; then + export CMAKE_OSX_ARCHITECTURES="x86_64;arm64" +else + echo "Unexpected architecture: $arch" + exit 1 +fi + +echo "=== (${PYTHON_VERSION}) Install Python build dependencies ===" +export PIP_SITE_PACKAGES=$(python -c 'import site; print(site.getsitepackages()[0])') +export PIP_TARGET_PLATFORM="macosx_${MACOSX_DEPLOYMENT_TARGET//./_}_${arch}" + +pip install \ + --only-binary=:all: \ + --target $PIP_SITE_PACKAGES \ + --platform $PIP_TARGET_PLATFORM \ + -r ${source_dir}/python/requirements-wheel-build.txt +pip install "delocate>=0.9" echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" : ${ARROW_DATASET:=ON} @@ -48,6 +70,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" : ${ARROW_PARQUET:=ON} : ${ARROW_PLASMA:=ON} : ${ARROW_S3:=ON} +: ${ARROW_SIMD_LEVEL:="SSE4_2"} : ${ARROW_TENSORFLOW:=ON} : ${ARROW_WITH_BROTLI:=ON} : ${ARROW_WITH_BZ2:=ON} @@ -57,19 +80,23 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ===" : ${ARROW_WITH_ZSTD:=ON} : ${CMAKE_BUILD_TYPE:=release} : ${CMAKE_GENERATOR:=Ninja} +: ${CMAKE_UNITY_BUILD:=ON} : ${VCPKG_FEATURE_FLAGS:=-manifests} : ${VCPKG_TARGET_TRIPLET:=${VCPKG_DEFAULT_TRIPLET:-x64-osx-static-${CMAKE_BUILD_TYPE}}} mkdir -p ${build_dir}/build pushd ${build_dir}/build + cmake \ -DARROW_BUILD_SHARED=ON \ + -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \ + -DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} \ -DARROW_BUILD_STATIC=OFF \ -DARROW_BUILD_TESTS=OFF \ -DARROW_DATASET=${ARROW_DATASET} \ -DARROW_DEPENDENCY_SOURCE="VCPKG" \ -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_FLIGHT==${ARROW_FLIGHT} \ + -DARROW_FLIGHT=${ARROW_FLIGHT} \ -DARROW_GANDIVA=${ARROW_GANDIVA} \ -DARROW_HDFS=${ARROW_HDFS} \ -DARROW_JEMALLOC=${ARROW_JEMALLOC} \ @@ -81,6 +108,7 @@ cmake \ -DARROW_PYTHON=ON \ -DARROW_RPATH_ORIGIN=ON \ -DARROW_S3=${ARROW_S3} \ + -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL} \ -DARROW_TENSORFLOW=${ARROW_TENSORFLOW} \ -DARROW_USE_CCACHE=ON \ -DARROW_WITH_BROTLI=${ARROW_WITH_BROTLI} \ @@ -92,7 +120,7 @@ cmake \ -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=${build_dir}/install \ - -DCMAKE_UNITY_BUILD=ON \ + -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \ -DOPENSSL_USE_STATIC_LIBS=ON \ -DVCPKG_MANIFEST_MODE=OFF \ -DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET} \ @@ -101,9 +129,6 @@ cmake \ cmake --build . --target install popd -# Check that we don't expose any unwanted symbols -# check_arrow_visibility - echo "=== (${PYTHON_VERSION}) Building wheel ===" export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE} export PYARROW_BUNDLE_ARROW_CPP=1 @@ -117,8 +142,11 @@ export PYARROW_WITH_ORC=${ARROW_ORC} export PYARROW_WITH_PARQUET=${ARROW_PARQUET} export PYARROW_WITH_PLASMA=${ARROW_PLASMA} export PYARROW_WITH_S3=${ARROW_S3} +export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}" # PyArrow build configuration export PKG_CONFIG_PATH=/usr/lib/pkgconfig:${build_dir}/install/lib/pkgconfig +# Set PyArrow version explicitly +export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION} pushd ${source_dir}/python python setup.py bdist_wheel @@ -127,7 +155,7 @@ popd echo "=== (${PYTHON_VERSION}) Show dynamic libraries the wheel depend on ===" deps=$(delocate-listdeps ${source_dir}/python/dist/*.whl) -if echo $deps | grep -v "^@rpath/lib\(arrow\|gandiva\|parquet\|plasma\)"; then +if echo $deps | grep -v "^pyarrow/lib\(arrow\|gandiva\|parquet\|plasma\)"; then echo "There are non-bundled shared library dependencies." exit 1 fi diff --git a/ci/scripts/python_wheel_macos_test.sh b/ci/scripts/python_wheel_macos_test.sh index 6ac8576d484..5dabf6e8c41 100755 --- a/ci/scripts/python_wheel_macos_test.sh +++ b/ci/scripts/python_wheel_macos_test.sh @@ -22,9 +22,11 @@ set -ex source_dir=${1} : ${ARROW_S3:=ON} +: ${ARROW_FLIGHT:=ON} export PYARROW_TEST_CYTHON=OFF export PYARROW_TEST_DATASET=ON +export PYARROW_TEST_FLIGHT=${ARROW_FLIGHT} export PYARROW_TEST_GANDIVA=OFF export PYARROW_TEST_HDFS=ON export PYARROW_TEST_ORC=ON @@ -33,7 +35,6 @@ export PYARROW_TEST_PARQUET=ON export PYARROW_TEST_PLASMA=ON export PYARROW_TEST_S3=${ARROW_S3} export PYARROW_TEST_TENSORFLOW=ON -export PYARROW_TEST_FLIGHT=ON export ARROW_TEST_DATA=${source_dir}/testing/data export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data @@ -47,7 +48,6 @@ import pyarrow import pyarrow._hdfs import pyarrow.csv import pyarrow.dataset -import pyarrow.flight import pyarrow.fs import pyarrow.json import pyarrow.orc @@ -58,6 +58,9 @@ import pyarrow.plasma if [ "${PYARROW_TEST_S3}" == "ON" ]; then python -c "import pyarrow._s3fs" fi +if [ "${PYARROW_TEST_FLIGHT}" == "ON" ]; then + python -c "import pyarrow.flight" +fi # Install testing dependencies pip install -r ${source_dir}/python/requirements-wheel-test.txt diff --git a/ci/vcpkg/arm64-osx-static-debug.cmake b/ci/vcpkg/arm64-osx-static-debug.cmake new file mode 100644 index 00000000000..6ed92b25b55 --- /dev/null +++ b/ci/vcpkg/arm64-osx-static-debug.cmake @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES arm64) + +set(VCPKG_BUILD_TYPE debug) diff --git a/ci/vcpkg/arm64-osx-static-release.cmake b/ci/vcpkg/arm64-osx-static-release.cmake new file mode 100644 index 00000000000..0aa78121602 --- /dev/null +++ b/ci/vcpkg/arm64-osx-static-release.cmake @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES arm64) + +set(VCPKG_BUILD_TYPE release) diff --git a/ci/vcpkg/universal2-osx-static-debug.cmake b/ci/vcpkg/universal2-osx-static-debug.cmake new file mode 100644 index 00000000000..7406ef3fe16 --- /dev/null +++ b/ci/vcpkg/universal2-osx-static-debug.cmake @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES "x86_64\;arm64") + +set(VCPKG_BUILD_TYPE debug) diff --git a/ci/vcpkg/universal2-osx-static-release.cmake b/ci/vcpkg/universal2-osx-static-release.cmake new file mode 100644 index 00000000000..0388ce78d0e --- /dev/null +++ b/ci/vcpkg/universal2-osx-static-release.cmake @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Darwin) +set(VCPKG_OSX_ARCHITECTURES "x86_64\;arm64") + +set(VCPKG_BUILD_TYPE release) diff --git a/cpp/cmake_modules/SetupCxxFlags.cmake b/cpp/cmake_modules/SetupCxxFlags.cmake index f12f071642b..aa8e5becab0 100644 --- a/cpp/cmake_modules/SetupCxxFlags.cmake +++ b/cpp/cmake_modules/SetupCxxFlags.cmake @@ -444,29 +444,31 @@ if(ARROW_CPU_FLAG STREQUAL "ppc") endif() if(ARROW_CPU_FLAG STREQUAL "armv8") - if(NOT CXX_SUPPORTS_ARMV8_ARCH) - message(FATAL_ERROR "Unsupported arch flag: ${ARROW_ARMV8_ARCH_FLAG}.") - endif() - if(ARROW_ARMV8_ARCH_FLAG MATCHES "native") - message(FATAL_ERROR "native arch not allowed, please specify arch explicitly.") - endif() - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${ARROW_ARMV8_ARCH_FLAG}") - if(NOT ARROW_SIMD_LEVEL STREQUAL "NONE") set(ARROW_HAVE_NEON ON) - add_definitions(-DARROW_HAVE_NEON) - endif() - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS - "5.4") - message(WARNING "Disable Armv8 CRC and Crypto as compiler doesn't support them well.") - else() - if(ARROW_ARMV8_ARCH_FLAG MATCHES "\\+crypto") - add_definitions(-DARROW_HAVE_ARMV8_CRYPTO) + if(NOT CXX_SUPPORTS_ARMV8_ARCH) + message(FATAL_ERROR "Unsupported arch flag: ${ARROW_ARMV8_ARCH_FLAG}.") + endif() + if(ARROW_ARMV8_ARCH_FLAG MATCHES "native") + message(FATAL_ERROR "native arch not allowed, please specify arch explicitly.") endif() - # armv8.1+ implies crc support - if(ARROW_ARMV8_ARCH_FLAG MATCHES "armv8\\.[1-9]|\\+crc") - add_definitions(-DARROW_HAVE_ARMV8_CRC) + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} ${ARROW_ARMV8_ARCH_FLAG}") + + add_definitions(-DARROW_HAVE_NEON) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS + "5.4") + message(WARNING "Disable Armv8 CRC and Crypto as compiler doesn't support them well." + ) + else() + if(ARROW_ARMV8_ARCH_FLAG MATCHES "\\+crypto") + add_definitions(-DARROW_HAVE_ARMV8_CRYPTO) + endif() + # armv8.1+ implies crc support + if(ARROW_ARMV8_ARCH_FLAG MATCHES "armv8\\.[1-9]|\\+crc") + add_definitions(-DARROW_HAVE_ARMV8_CRC) + endif() endif() endif() endif() diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 39ccbbe72b0..3d3df156e04 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1562,9 +1562,9 @@ if(ARROW_MIMALLOC) endif() set(MIMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/mimalloc_ep/src/mimalloc_ep") - set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_PREFIX}/lib/mimalloc-1.6/include") + set(MIMALLOC_INCLUDE_DIR "${MIMALLOC_PREFIX}/include/mimalloc-1.7") set(MIMALLOC_STATIC_LIB - "${MIMALLOC_PREFIX}/lib/mimalloc-1.6/${CMAKE_STATIC_LIBRARY_PREFIX}${MIMALLOC_LIB_BASE_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + "${MIMALLOC_PREFIX}/lib/mimalloc-1.7/${CMAKE_STATIC_LIBRARY_PREFIX}${MIMALLOC_LIB_BASE_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) set(MIMALLOC_CMAKE_ARGS diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index a2f7f2c7213..e0cbb2a6a12 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -42,7 +42,7 @@ ARROW_JEMALLOC_BUILD_VERSION=5.2.1 ARROW_LZ4_BUILD_VERSION=v1.9.3 # mimalloc 1.6.7 didn't build on Visual Studio 2015 # https://github.com/microsoft/mimalloc/issues/353 -ARROW_MIMALLOC_BUILD_VERSION=v1.6.4 +ARROW_MIMALLOC_BUILD_VERSION=v1.7.2 ARROW_ORC_BUILD_VERSION=1.6.6 ARROW_PROTOBUF_BUILD_VERSION=v3.14.0 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py index 01571b43c95..c6b4a6473b8 100644 --- a/dev/archery/archery/docker/cli.py +++ b/dev/archery/archery/docker/cli.py @@ -116,7 +116,8 @@ def docker_build(obj, image, *, force_pull, using_docker_cli, compose.build(image, use_cache=use_cache, use_leaf_cache=use_leaf_cache, using_docker=using_docker_cli, - using_buildx=using_docker_buildx) + using_buildx=using_docker_buildx, + pull_parents=force_pull) except UndefinedImage as e: raise click.ClickException( "There is no service/image defined in docker-compose.yml with " diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index 2fe6e353ccc..aaf16bdfa6e 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -245,7 +245,7 @@ def _pull(service): _pull(service) def build(self, service_name, use_cache=True, use_leaf_cache=True, - using_docker=False, using_buildx=False): + using_docker=False, using_buildx=False, pull_parents=True): def _build(service, use_cache): if 'build' not in service: # nothing to do @@ -253,7 +253,7 @@ def _build(service, use_cache): args = [] cache_from = list(service.get('build', {}).get('cache_from', [])) - if use_cache: + if pull_parents: for image in cache_from: if image not in self.pull_memory: try: @@ -262,7 +262,8 @@ def _build(service, use_cache): print(e) finally: self.pull_memory.add(image) - else: + + if not use_cache: args.append('--no-cache') # turn on inline build cache, this is a docker buildx feature diff --git a/dev/archery/setup.py b/dev/archery/setup.py index eb70551de52..66480737547 100755 --- a/dev/archery/setup.py +++ b/dev/archery/setup.py @@ -34,6 +34,8 @@ 'release': [jinja_req, 'jira', 'semver', 'gitpython'], 'crossbow': ['github3.py', jinja_req, 'pygit2>=1.6.0', 'ruamel.yaml', 'setuptools_scm'], + 'crossbow-upload': ['github3.py', jinja_req, 'ruamel.yaml', + 'setuptools_scm'], } extras['bot'] = extras['crossbow'] + ['pygithub', 'jira'] extras['all'] = list(set(functools.reduce(operator.add, extras.values()))) diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja index 5f1056ca6a0..be265caa48a 100644 --- a/dev/tasks/macros.jinja +++ b/dev/tasks/macros.jinja @@ -69,7 +69,7 @@ on: python-version: 3.8 - name: Setup Crossbow shell: bash - run: pip install -e arrow/dev/archery[crossbow] + run: pip install -e arrow/dev/archery[crossbow-upload] - name: Upload artifacts shell: bash run: | @@ -116,7 +116,7 @@ on: - task: UsePythonVersion@0 inputs: versionSpec: '3.8' - - script: pip install -e arrow/dev/archery[crossbow] + - script: pip install -e arrow/dev/archery[crossbow-upload] displayName: Install Crossbow - bash: | archery crossbow \ @@ -170,7 +170,7 @@ on: {%- macro travis_upload_releases(pattern) -%} - sudo -H pip3 install pygit2==1.0 - - sudo -H pip3 install -e arrow/dev/archery[crossbow] + - sudo -H pip3 install -e arrow/dev/archery[crossbow-upload] - | archery crossbow \ --queue-path $(pwd) \ diff --git a/dev/tasks/python-wheels/github.osx.yml b/dev/tasks/python-wheels/github.osx.amd64.yml similarity index 67% rename from dev/tasks/python-wheels/github.osx.yml rename to dev/tasks/python-wheels/github.osx.amd64.yml index a2e5c0af21b..863bd7fa9c1 100644 --- a/dev/tasks/python-wheels/github.osx.yml +++ b/dev/tasks/python-wheels/github.osx.amd64.yml @@ -20,37 +20,40 @@ env: ARROW_S3: {{ arrow_s3 }} - MACOSX_DEPLOYMENT_TARGET: {{ macos_deployment_target }} - MB_PYTHON_VERSION: {{ python_version }} - PLAT: x86_64 + CC: "clang" + CXX: "clang++" + MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" PYARROW_BUILD_VERBOSE: 1 - PYARROW_VERSION: {{ arrow.no_rc_version }} - PYTHON_VERSION: {{ python_version }} - SETUPTOOLS_SCM_PRETEND_VERSION: {{ arrow.no_rc_version }} + PYARROW_VERSION: "{{ arrow.no_rc_version }}" + PYTHON_VERSION: "{{ python_version }}" + PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" VCPKG_DEFAULT_TRIPLET: x64-osx-static-release VCPKG_FEATURE_FLAGS: "-manifests" VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} + VCPKG_VERSION: "{{ vcpkg_version }}" jobs: build: name: Build wheel for OS X - runs-on: macos-latest + runs-on: macos-10.15 steps: {{ macros.github_checkout_arrow()|indent }} - name: Install System Dependencies - run: brew install bison coreutils ninja cmake + run: brew install bash bison coreutils ninja cmake - uses: actions/cache@v2 id: vcpkg-cache with: path: vcpkg - key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/**') }}" }} + key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }} - name: Install Vcpkg if: steps.vcpkg-cache.outputs.cache-hit != 'true' shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "10.15" run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT - name: Install Packages @@ -83,35 +86,24 @@ jobs: run: $VCPKG_ROOT/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] {% endif %} - - name: Setup Multibuild - run: | - git clone https://github.com/matthew-brett/multibuild - git -C multibuild checkout 03950c9a7feb09d215f82d6563c4ffd91274a1e1 + - name: Install Python {{ python_version }} + shell: bash + run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }} - name: Build Wheel - env: - CONFIG_PATH: /dev/null + shell: bash run: | - # configure environment and install python - source multibuild/common_utils.sh - source multibuild/travis_osx_steps.sh - before_install + $PYTHON -m virtualenv build-env + source build-env/bin/activate + pip install --upgrade pip wheel + arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build - # install python dependencies - pip install -r arrow/python/requirements-wheel-build.txt delocate - - # build the wheel - arrow/ci/scripts/python_wheel_macos_build.sh $(pwd)/arrow $(pwd)/build - - - name: Setup Python for Testing - uses: actions/setup-python@v2 - with: - python-version: "{{ python_version }}" - - - name: Test the Wheel + - name: Test Wheel + shell: bash run: | - # TODO(kszucs): temporarily remove homebrew libs - unset MACOSX_DEPLOYMENT_TARGET + $PYTHON -m virtualenv test-env + source test-env/bin/activate + pip install --upgrade pip wheel arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} diff --git a/dev/tasks/python-wheels/github.osx.arm64.yml b/dev/tasks/python-wheels/github.osx.arm64.yml new file mode 100644 index 00000000000..4fa95bbefdc --- /dev/null +++ b/dev/tasks/python-wheels/github.osx.arm64.yml @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prerequisits on the host: +# - brew install bash bison coreutils ninja cmake +# - sudo arrow/ci/scripts/install_python.sh macos 3.9 + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +env: + ARROW_FLIGHT: OFF + ARROW_JEMALLOC: OFF + ARROW_SIMD_LEVEL: NONE + CC: "clang" + CMAKE_BUILD_TYPE: release + CMAKE_CXX_COMPILER_LAUNCHER: "ccache" + CXX: "clang++" + MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" + PYARROW_BUILD_VERBOSE: 1 + PYARROW_VERSION: "{{ arrow.no_rc_version }}" + PYTHON_VERSION: "{{ python_version }}" + PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" + VCPKG_DEFAULT_TRIPLET: {{ arch }}-osx-static-release + VCPKG_FEATURE_FLAGS: "-manifests" + VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} + VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} + VCPKG_VERSION: "{{ vcpkg_version }}" + +jobs: + build: + name: Build wheel for OS X + runs-on: self-hosted + steps: + - name: Cleanup + shell: bash + run: rm -rf arrow vcpkg build crossbow-env build-env test-*-env + + {{ macros.github_checkout_arrow()|indent }} + + - name: Add Brew's Bison to PATH + shell: bash + run: echo "/opt/homebrew/opt/bison/bin" >> $GITHUB_PATH + + - name: Install Vcpkg + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "11.0" + run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT + + - name: Install OpenSSL + shell: bash + run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl + + {% if arch == "universal2" %} + # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is + # unable to propagate the list of architectures from VCPKG_OSX_ARCHIETCTURES. + # In order to prevent link time warnings (which may turn out to be errors) + # we compile OpenSSL separately for the two architectures and merge the + # binaries into universal2 ones using `lipo`. + - name: Create universal binaries for OpenSSL + shell: bash + run: | + for arch in arm64 x64; do + VCPKG_DEFAULT_TRIPLET=${arch}-osx-static-release arch -arm64 $VCPKG_ROOT/vcpkg install openssl + done + for lib in libcrypto libssl; do + lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \ + $VCPKG_ROOT/installed/x64-osx-static-release/lib/${lib}.a \ + -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a + done + {% endif %} + + - name: Install Packages + run: | + arch -arm64 $VCPKG_ROOT/vcpkg install \ + aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ + boost-filesystem \ + brotli \ + bzip2 \ + c-ares \ + curl \ + flatbuffers \ + gflags \ + glog \ + lz4 \ + orc \ + protobuf \ + rapidjson \ + re2 \ + snappy \ + thrift \ + utf8proc \ + zlib \ + zstd + + - name: Build Wheel + shell: bash + run: | + $PYTHON -m virtualenv build-env + source build-env/bin/activate + pip install --upgrade pip wheel + arch -arm64 arrow/ci/scripts/python_wheel_macos_build.sh {{ arch }} $(pwd)/arrow $(pwd)/build + + - name: Test Wheel on ARM64 + shell: bash + run: | + $PYTHON -m virtualenv test-arm64-env + source test-arm64-env/bin/activate + pip install --upgrade pip wheel + arch -arm64 arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow + + {% if arch == "universal2" %} + - name: Test Wheel on AMD64 + shell: bash + run: | + $PYTHON -m virtualenv test-amd64-env + source test-amd64-env/bin/activate + pip install --upgrade pip wheel + arch -x86_64 arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow + {% endif %} + + - name: Upload artifacts + shell: bash + run: | + $PYTHON -m virtualenv crossbow-env + source crossbow-env/bin/activate + arch -arm64 pip install -e arrow/dev/archery[crossbow-upload] + arch -arm64 archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + "arrow/python/dist/*.whl" + env: + CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }} + + {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 7aea18fdd83..ea0571f3989 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -355,9 +355,9 @@ tasks: ("10.13", "high-sierra", "ON")] %} {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %} - wheel-osx-{{ macos_codename }}-{{ python_tag }}: + wheel-macos-{{ macos_codename }}-{{ python_tag }}-amd64: ci: github - template: python-wheels/github.osx.yml + template: python-wheels/github.osx.amd64.yml params: vcpkg_version: "2021.04.30" python_version: {{ python_version }} @@ -370,7 +370,7 @@ tasks: {############################## Wheel Windows ################################} - wheel-windows-{{ python_tag }}: + wheel-windows-{{ python_tag }}-amd64: ci: github template: python-wheels/github.windows.yml params: @@ -380,6 +380,30 @@ tasks: {% endfor %} +{############################## Wheel OSX M1 #################################} + + wheel-macos-big-sur-cp39-arm64: + ci: github + template: python-wheels/github.osx.arm64.yml + params: + arch: arm64 + vcpkg_version: "2021.04.30" + python_version: "3.9" + macos_deployment_target: "11.0" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_11_0_arm64.whl + + wheel-macos-big-sur-cp39-universal2: + ci: github + template: python-wheels/github.osx.arm64.yml + params: + arch: universal2 + vcpkg_version: "2021.04.30" + python_version: "3.9" + macos_deployment_target: "11.0" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_11_0_universal2.whl + {############################ Python sdist ####################################} python-sdist: diff --git a/docker-compose.yml b/docker-compose.yml index 93bba30772b..84ce4dac1ba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -810,6 +810,16 @@ services: command: /arrow/ci/scripts/python_wheel_manylinux_test.sh unittests python-wheel-windows-vs2017: + # The windows images must be built locally and pushed to a remote registry: + # export REPO=ghcr.io/ursacomputing/arrow + # PYTHON=3.6 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 + # PYTHON=3.7 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 + # PYTHON=3.8 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 + # PYTHON=3.9 archery docker build --no-pull --using-docker-cli python-wheel-windows-vs2017 + # PYTHON=3.6 archery docker push python-wheel-windows-vs2017 + # PYTHON=3.7 archery docker push python-wheel-windows-vs2017 + # PYTHON=3.8 archery docker push python-wheel-windows-vs2017 + # PYTHON=3.9 archery docker push python-wheel-windows-vs2017 image: ${REPO}:python-${PYTHON}-wheel-windows-vs2017-vcpkg-${VCPKG} build: args: diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 74a352e8a5b..b2878d2971c 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,6 +1,11 @@ cython>=0.29.11 -numpy==1.16.6; python_version < "3.9" and platform_machine != "aarch64" -numpy==1.19.4; python_version >= "3.9" or platform_machine == "aarch64" -pandas<1.1.0; python_version < "3.8" setuptools_scm wheel +numpy==1.19.4; platform_system == "Linux" and platform_machine == "aarch64" +numpy==1.16.6; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" +numpy==1.19.4; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" +numpy==1.21.0; platform_system == "Darwin" and platform_machine == "arm64" +numpy==1.16.6; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.8" +numpy==1.19.4; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.8" +numpy==1.16.6; platform_system == "Windows" and python_version < "3.9" +numpy==1.19.4; platform_system == "Windows" and python_version >= "3.9" diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 7377f6b6d8f..723dbdff76c 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -1,11 +1,26 @@ cffi cython hypothesis -numpy==1.19.4; python_version != "3.9" or platform_machine != "aarch64" -numpy==1.20.1; python_version == "3.9" and platform_machine == "aarch64" -pandas<1.1.0; python_version < "3.8" and platform_machine != "aarch64" -pandas; python_version >= "3.8" or platform_machine == "aarch64" -pickle5; (python_version == "3.6" or python_version == "3.7") and sys_platform != "win32" +pickle5; platform_system != "Windows" and python_version < "3.8" pytest pytest-lazy-fixture pytz + +numpy==1.19.5; platform_system == "Linux" and platform_machine == "aarch64" and python_version < "3.7" +numpy==1.20.3; platform_system == "Linux" and platform_machine == "aarch64" and python_version >= "3.7" +numpy==1.19.5; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.9" +numpy==1.20.3; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.9" +numpy==1.21.0; platform_system == "Darwin" and platform_machine == "arm64" +numpy==1.19.5; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.9" +numpy==1.20.3; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.9" +numpy==1.19.5; platform_system == "Windows" and python_version < "3.9" +numpy==1.20.3; platform_system == "Windows" and python_version >= "3.9" + +pandas<1.1.0; platform_system == "Linux" and platform_machine != "aarch64" and python_version < "3.8" +pandas; platform_system == "Linux" and platform_machine != "aarch64" and python_version >= "3.8" +pandas; platform_system == "Linux" and platform_machine == "aarch64" +pandas<1.1.0; platform_system == "Darwin" and platform_machine != "arm64" and python_version < "3.8" +pandas; platform_system == "Darwin" and platform_machine != "arm64" and python_version >= "3.8" +pandas; platform_system == "Darwin" and platform_machine == "arm64" +pandas<1.1.0; platform_system == "Windows" and python_version < "3.8" +pandas; platform_system == "Windows" and python_version >= "3.8"