diff --git a/cpp/Brewfile b/cpp/Brewfile index 7de6c7deabe..2b7026b00a0 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -25,7 +25,6 @@ brew "flatbuffers" brew "git" brew "glog" brew "grpc" -brew "llvm" brew "llvm@8" brew "lz4" brew "minio" diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d084f346803..a5d85f5263d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -768,6 +768,8 @@ if(MSVC AND ARROW_USE_STATIC_CRT) set(Boost_USE_STATIC_RUNTIME ON) endif() set(Boost_ADDITIONAL_VERSIONS + "1.74.0" + "1.74" "1.73.0" "1.73" "1.72.0" @@ -1344,6 +1346,7 @@ macro(build_protobuf) -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_LIBDIR=lib "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}" + -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX=) if(ZLIB_ROOT) @@ -2358,6 +2361,8 @@ macro(build_grpc) if(GFLAGS_VENDORED) add_dependencies(grpc_dependencies gflags_ep) endif() + resolve_dependency(RE2) + add_dependencies(grpc_dependencies re2_ep) add_dependencies(grpc_dependencies ${ARROW_PROTOBUF_LIBPROTOBUF} c-ares::cares ZLIB::ZLIB) @@ -2407,6 +2412,12 @@ macro(build_grpc) -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package + -DgRPC_PROTOBUF_PACKAGE_TYPE=CONFIG + -DgRPC_MSVC_STATIC_RUNTIME=OFF + -DgRPC_RE2_PROVIDER=package + -DRE2_ROOT_DIR=${RE2_PREFIX} + -Dre2_ROOT=${RE2_PREFIX} + -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_CXX_FLAGS=${GRPC_CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${EP_C_FLAGS} -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 545943d30c6..62dc5077e0f 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -35,13 +35,13 @@ ARROW_CARES_BUILD_VERSION=1.16.1 ARROW_GBENCHMARK_BUILD_VERSION=v1.5.1 ARROW_GFLAGS_BUILD_VERSION=v2.2.2 ARROW_GLOG_BUILD_VERSION=v0.4.0 -ARROW_GRPC_BUILD_VERSION=v1.29.1 +ARROW_GRPC_BUILD_VERSION=v1.33.2 ARROW_GTEST_BUILD_VERSION=1.8.1 ARROW_JEMALLOC_BUILD_VERSION=5.2.1 ARROW_LZ4_BUILD_VERSION=v1.9.2 ARROW_MIMALLOC_BUILD_VERSION=v1.6.4 ARROW_ORC_BUILD_VERSION=1.6.2 -ARROW_PROTOBUF_BUILD_VERSION=v3.12.1 +ARROW_PROTOBUF_BUILD_VERSION=v3.13.0 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require # a pre-release version of RapidJSON to build with GCC 8 without # warnings. diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 0cb9e1b8b11..9a903e91bf2 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -634,7 +634,7 @@ IMPORT_TESTS } test_linux_wheels() { - local py_arches="3.5m 3.6m 3.7m 3.8" + local py_arches="3.5m 3.6m 3.7m 3.8 3.9" local manylinuxes="1 2010 2014" for py_arch in ${py_arches}; do @@ -659,7 +659,7 @@ test_linux_wheels() { } test_macos_wheels() { - local py_arches="3.5m 3.6m 3.7m 3.8" + local py_arches="3.5m 3.6m 3.7m 3.8 3.9" for py_arch in ${py_arches}; do local env=_verify_wheel-${py_arch} diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml index 47122636a58..f2d3ceaac68 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml @@ -1,7 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -9,7 +9,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '7.5' +- '7' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7.5' +- '7' docker_image: - condaforge/linux-anvil-aarch64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -39,7 +39,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -55,11 +55,16 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml index 1ce4521a92a..611c39c907c 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml @@ -1,7 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -9,7 +9,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '7.5' +- '7' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7.5' +- '7' docker_image: - condaforge/linux-anvil-aarch64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -39,7 +39,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -55,11 +55,16 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml index 9bf11fd5215..2f0fc0e2306 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml @@ -1,7 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -9,7 +9,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '7.5' +- '7' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7.5' +- '7' docker_image: - condaforge/linux-anvil-aarch64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -39,7 +39,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -55,11 +55,16 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml new file mode 100644 index 00000000000..2ec87205a0f --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml @@ -0,0 +1,71 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '7' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-aarch64 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml index 05234c57e34..4c5061d6ff2 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml index e4ab58abee6..db820f3ff26 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml index e15e3bf7623..24810f3c0f1 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml new file mode 100644 index 00000000000..ebe422be367 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml @@ -0,0 +1,69 @@ +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '9.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:9.2 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - docker_image +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml index e2952d8375d..358814fb0da 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml index 0da1784bdd6..054a730ee50 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml index 8fc3382d8a9..a3bfc58ea8f 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml new file mode 100644 index 00000000000..88541395052 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml @@ -0,0 +1,69 @@ +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - docker_image +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml index e4d163f526c..c019508dce4 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml @@ -1,7 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -25,7 +25,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ macos_machine: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,11 +51,16 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml index 3a805335574..8f66d609797 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml @@ -1,7 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -25,7 +25,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ macos_machine: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -49,13 +49,18 @@ pin_run_as_build: zlib: max_pin: x.x python: -- 3.7.* *_cpython +- 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml index 9e7aa65b8df..ad449921253 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml @@ -33,7 +33,7 @@ lz4_c: macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.16' +- '1.17.3' orc: - 1.6.4 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml new file mode 100644 index 00000000000..c630217d111 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml @@ -0,0 +1,67 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '10' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '10' +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +macos_machine: +- x86_64-apple-darwin13.4.0 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml index bce682a27af..40199c679de 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -19,7 +19,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -41,11 +41,14 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - win-64 +zip_keys: +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml index 8b74b5d79a7..88d17108a98 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -19,7 +19,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -41,11 +41,14 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - win-64 +zip_keys: +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml index 32cbf60e952..e18785de135 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -19,7 +19,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -41,11 +41,14 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - win-64 +zip_keys: +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml new file mode 100644 index 00000000000..6177f96ce01 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml @@ -0,0 +1,55 @@ +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- vs2017 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- win-64 +zip_keys: +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 272ed8c5382..cdbfc5d5b4d 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -179,7 +179,7 @@ outputs: host: - {{ pin_subpackage('arrow-cpp', exact=True) }} - cython - - numpy 1.16.* + - numpy - python - setuptools - setuptools_scm @@ -244,7 +244,7 @@ outputs: - {{ pin_subpackage('arrow-cpp', exact=True) }} - {{ pin_subpackage('pyarrow', exact=True) }} - cython - - numpy 1.16.* + - numpy - python - setuptools - setuptools_scm diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index 1bc9e0742d7..c5851ed73ef 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -42,7 +42,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Setup Archery run: pip install -e arrow/dev/archery[docker] - name: Execute Docker Build diff --git a/dev/tasks/python-wheels/github.win.yml b/dev/tasks/python-wheels/github.win.yml index f7638d8be36..17da17b73d7 100644 --- a/dev/tasks/python-wheels/github.win.yml +++ b/dev/tasks/python-wheels/github.win.yml @@ -53,9 +53,9 @@ jobs: run: | echo "c:\\Miniconda\\condabin" >> $GITHUB_PATH - name: Build wheel - shell: bash + shell: cmd run: | - arrow/dev/tasks/python-wheels/win-build.bat + call arrow/dev/tasks/python-wheels/win-build.bat - name: Prepare artifacts # the artifacts must be uploaded from a directory relative to the build root shell: bash diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index e029932ecf6..e6bd6159de9 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -50,7 +50,10 @@ function build_wheel { export ARROW_HOME=`pwd`/arrow-dist export PARQUET_HOME=`pwd`/arrow-dist - pip install $(pip_opts) -r python/requirements-wheel-build.txt + # If NumPy builds from the source make sure it is built against OpenBLAS + # See: https://github.com/numpy/numpy/issues/15947#issuecomment-686159427 + OPENBLAS=$(brew --prefix openblas) pip install \ + $(pip_opts) -r python/requirements-wheel-build.txt git submodule update --init export ARROW_TEST_DATA=`pwd`/testing/data diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index 99c667a6450..a67f36443c7 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -29,6 +29,7 @@ addons: - flatbuffers - flex - git + - openblas - openssl@1.1 - protobuf - python@3.8 @@ -70,7 +71,15 @@ install: # test the built wheels, move Homebrew directory to ensure # things are properly statically-linked - - sudo mv $(brew --cellar){,.bak} + - | + CELLAR=$(brew --cellar) + sudo mv $CELLAR{,.bak} + + # make a new Cellar, allowing numpy to dynamically link to openblas + sudo mkdir $CELLAR + sudo ln -s ../Cellar.bak/openblas $CELLAR/openblas + # and brew's openblas will need gcc + sudo ln -s ../Cellar.bak/gcc $CELLAR/gcc # install the built wheel and test dependencies - install_wheel arrow # run unit tests before removing the system libraries @@ -78,7 +87,11 @@ install: # run the import tests - run_import_tests - - sudo mv $(brew --cellar){.bak,} + - | + CELLAR=$(brew --cellar) + sudo rm $CELLAR/{gcc,openblas} + sudo rmdir $CELLAR + sudo mv $CELLAR{.bak,} # before_install activates a virtualenv but we need the system python3 - deactivate diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 9d02e787f0f..3e875fe3649 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -24,13 +24,12 @@ @rem create conda environment for compiling call conda.bat create -n wheel-build -q -y -c conda-forge ^ - --file=arrow\ci\conda_env_cpp.yml ^ - --file=arrow\ci\conda_env_python.yml ^ - "vs2015_runtime<14.16" ^ + cmake ninja pkg-config numpy=1.19 ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build +set CONDA_PREFIX_CMAKE_STYLE=%CONDA_PREFIX:\=/% set ARROW_HOME=%CONDA_PREFIX%\Library set PARQUET_HOME=%CONDA_PREFIX%\Library echo %ARROW_HOME% @@ -44,35 +43,38 @@ pushd arrow\cpp\build @rem conda-forge also results in a broken build so we use the BUNDLED @rem dependency resolution strategy for now +@rem -DARROW_DEPENDENCY_SOURCE=BUNDLED is required because recent +@rem conda-forge packages don't provide static library. cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DARROW_BOOST_USE_SHARED=OFF ^ -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=OFF ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_DEPENDENCY_SOURCE=CONDA ^ - -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ -DARROW_CXXFLAGS="/MP" ^ - -DARROW_WITH_BZ2=OFF ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_BROTLI=ON ^ -DARROW_DATASET=ON ^ + -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ -DARROW_FLIGHT=ON ^ -DARROW_GANDIVA=OFF ^ - -DARROW_LZ4_USE_SHARED=OFF ^ -DARROW_MIMALLOC=ON ^ + -DARROW_OPENSSL_USE_SHARED=OFF ^ + -DARROW_PROTOBUF_USE_SHARED=OFF ^ -DARROW_PARQUET=ON ^ -DARROW_PYTHON=ON ^ - -DARROW_SNAPPY_USE_SHARED=OFF ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=OFF ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ - -DBrotli_SOURCE=BUNDLED ^ - -Dzstd_SOURCE=BUNDLED ^ - -Dutf8proc_SOURCE=BUNDLED ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DOPENSSL_ROOT_DIR=%CONDA_PREFIX_CMAKE_STYLE%/Library ^ .. || exit /B +cmake ^ + --build . ^ + --config Release ^ + --parallel %NUMBER_OF_PROCESSORS% ^ + --target grpc_ep || exit /B cmake ^ --build . ^ --config Release ^ @@ -114,6 +116,6 @@ python -c "import pyarrow.flight" || exit /B python -c "import pyarrow.dataset" || exit /B @rem run the python tests, but disable the cython because there is a linking -@rem issue on python 3.8 +@rem issue on python 3.9 set PYARROW_TEST_CYTHON=OFF python -m pytest -rs --pyargs pyarrow || exit /B diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a2349f5b5a3..603ebd82c48 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -174,6 +174,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-linux-gcc-py39-cpu: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_cuda_compiler_versionNonepython3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-linux-gcc-py36-cuda: ci: azure template: conda-recipes/azure.linux.yml @@ -201,6 +210,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 + conda-linux-gcc-py39-cuda: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_cuda_compiler_version9.2python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 + conda-linux-gcc-py36-aarch64: ci: drone template: conda-recipes/drone.yml @@ -228,6 +246,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-linux-gcc-py39-aarch64: + ci: drone + template: conda-recipes/drone.yml + params: + config: linux_aarch64_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Conda OSX #################################### conda-osx-clang-py36: @@ -259,6 +286,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-osx-clang-py39: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Conda Windows ################################ conda-win-vs2017-py36: @@ -290,6 +326,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-win-vs2017-py39: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Wheel Linux ################################## wheel-manylinux1-cp35m: @@ -348,6 +393,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-manylinux1_x86_64.whl + wheel-manylinux1-cp39: + ci: azure + template: python-wheels/azure.linux.yml + params: + python_version: 3.9 + unicode_width: 16 + wheel_tag: manylinux1 + wheel_dir: manylinux1 + test_docker_images: + - python:3.9 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-manylinux1_x86_64.whl + wheel-manylinux2010-cp35m: ci: azure template: python-wheels/azure.linux.yml @@ -404,6 +463,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-manylinux2010_x86_64.whl + wheel-manylinux2010-cp39: + ci: azure + template: python-wheels/azure.linux.yml + params: + python_version: 3.9 + unicode_width: 16 + wheel_tag: manylinux2010 + wheel_dir: manylinux201x + test_docker_images: + - python:3.9 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-manylinux2010_x86_64.whl + wheel-manylinux2014-cp35m: ci: azure template: python-wheels/azure.linux.yml @@ -460,6 +533,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-manylinux2014_x86_64.whl + wheel-manylinux2014-cp39: + ci: azure + template: python-wheels/azure.linux.yml + params: + python_version: 3.9 + unicode_width: 16 + wheel_tag: manylinux2014 + wheel_dir: manylinux201x + test_docker_images: + - python:3.9 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-manylinux2014_x86_64.whl + ############################## Wheel OSX #################################### wheel-osx-mavericks-cp35m: @@ -502,6 +589,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl + wheel-osx-mavericks-cp39: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.9 + macos_deployment_target: 10.9 + arrow_s3: "OFF" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_9_x86_64.whl + # enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl wheel-osx-high-sierra-cp35m: @@ -544,6 +641,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_13_x86_64.whl + wheel-osx-high-sierra-cp39: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.9 + macos_deployment_target: 10.13 + arrow_s3: "ON" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_13_x86_64.whl + ############################## Wheel Windows ################################ wheel-win-cp36m: @@ -570,6 +677,14 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-win_amd64.whl + wheel-win-cp39: + ci: github + template: python-wheels/github.win.yml + params: + python_version: 3.9 + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-win_amd64.whl + ############################## Linux PKGS #################################### debian-stretch-amd64: diff --git a/docker-compose.yml b/docker-compose.yml index 195fc0104d2..4424943300b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -473,7 +473,7 @@ services: # docker-compose run --rm conda-python # Parameters: # ARCH: amd64, arm32v7 - # PYTHON: 3.6, 3.7, 3.8 + # PYTHON: 3.6, 3.7, 3.8, 3.9 image: ${REPO}:${ARCH}-conda-python-${PYTHON} build: context: . diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base index 15487e02fbc..8bc9cd2eb0f 100644 --- a/python/manylinux1/Dockerfile-x86_64_base +++ b/python/manylinux1/Dockerfile-x86_64_base @@ -17,7 +17,7 @@ # See https://quay.io/repository/pypa/manylinux1_x86_64?tab=tags # to update base image. -FROM quay.io/pypa/manylinux1_x86_64:2020-03-07-9c5ba95 +FROM quay.io/pypa/manylinux1_x86_64:2020-10-06-c4a4a7c # Install dependencies RUN yum install -y xz ccache flex && yum clean all @@ -89,7 +89,7 @@ RUN /build_glog.sh ENV GLOG_HOME /usr/local WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 # Remove unneeded Python versions RUN rm -rf /opt/_internal/cpython-2.7.* diff --git a/python/manylinux1/Dockerfile-x86_64_ubuntu b/python/manylinux1/Dockerfile-x86_64_ubuntu index b038f32af2a..290fbc1a68b 100644 --- a/python/manylinux1/Dockerfile-x86_64_ubuntu +++ b/python/manylinux1/Dockerfile-x86_64_ubuntu @@ -34,7 +34,7 @@ ADD scripts/install_cmake.sh / RUN /install_cmake.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 ADD scripts/build_openssl.sh / RUN /build_openssl.sh diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index ab67fced573..0bdd2e13ecf 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -62,7 +62,7 @@ export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost - mkdir -p /io/dist # Must pass PYTHON_VERSION env variable -# possible values are: 3.5 3.6 3.7 3.8 +# possible values are: 3.5 3.6 3.7 3.8 3.9 UNICODE_WIDTH=32 # Dummy value, irrelevant for Python 3 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" diff --git a/python/manylinux201x/Dockerfile-x86_64_base_2010 b/python/manylinux201x/Dockerfile-x86_64_base_2010 index 0d7d497ef5e..99883b711bd 100644 --- a/python/manylinux201x/Dockerfile-x86_64_base_2010 +++ b/python/manylinux201x/Dockerfile-x86_64_base_2010 @@ -17,7 +17,7 @@ # See https://quay.io/repository/pypa/manylinux2010_x86_64?tab=tags # to update base image. -FROM quay.io/pypa/manylinux2010_x86_64:2020-03-07-1825e8f +FROM quay.io/pypa/manylinux2010_x86_64:2020-10-06-786a931 # Install build dependencies RUN yum install -y xz bison ccache flex wget @@ -32,7 +32,7 @@ ADD scripts/build_zlib.sh / RUN /build_zlib.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 # Remove unneeded Python versions RUN rm -rf /opt/_internal/cpython-2.7.* diff --git a/python/manylinux201x/Dockerfile-x86_64_base_2014 b/python/manylinux201x/Dockerfile-x86_64_base_2014 index d8c71ad9c7f..93bafc40f8b 100644 --- a/python/manylinux201x/Dockerfile-x86_64_base_2014 +++ b/python/manylinux201x/Dockerfile-x86_64_base_2014 @@ -17,7 +17,7 @@ # See https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags # to update base image. -FROM quay.io/pypa/manylinux2014_x86_64:2020-03-07-06139da +FROM quay.io/pypa/manylinux2014_x86_64:2020-10-06-b2ca7a1 # Install build dependencies RUN yum install -y xz bison ccache flex wget @@ -32,7 +32,7 @@ ADD scripts/build_zlib.sh / RUN /build_zlib.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 # Remove unneeded Python versions RUN rm -rf /opt/_internal/cpython-2.7.* diff --git a/python/manylinux201x/build_arrow.sh b/python/manylinux201x/build_arrow.sh index 3913b46b680..41586737991 100755 --- a/python/manylinux201x/build_arrow.sh +++ b/python/manylinux201x/build_arrow.sh @@ -62,7 +62,7 @@ export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig mkdir -p /io/dist # Must pass PYTHON_VERSION env variable -# possible values are: 3.5 3.6 3.7 3.8 +# possible values are: 3.5 3.6 3.7 3.8 3.9 UNICODE_WIDTH=32 # Dummy value, irrelevant for Python 3 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" diff --git a/python/manylinux201x/scripts/install_cmake.sh b/python/manylinux201x/scripts/install_cmake.sh index 2afef8e70c9..474b72ecc75 100755 --- a/python/manylinux201x/scripts/install_cmake.sh +++ b/python/manylinux201x/scripts/install_cmake.sh @@ -18,6 +18,6 @@ export CMAKE_VERSION=3.18.2.post1 /opt/python/cp37-cp37m/bin/pip install cmake==${CMAKE_VERSION} ninja -ln -s /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake -ln -s /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja +ln -fs /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake +ln -fs /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/* diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index f46e75fc3ac..44b62a95f40 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -1424,6 +1424,8 @@ def test_array_from_pandas_date_with_mask(self): expected = pd.Series([None, date(1991, 1, 1), None]) assert pa.Array.from_pandas(expected).equals(result) + @pytest.mark.skipif('1.16.0' <= LooseVersion(np.__version__) < '1.16.1', + reason='Until numpy/numpy#12745 is resolved') def test_fixed_offset_timezone(self): df = pd.DataFrame({ 'a': [ @@ -2787,7 +2789,7 @@ def _check_serialize_components_roundtrip(pd_obj): tm.assert_series_equal(pd_obj, deserialized) -@pytest.mark.skipif(LooseVersion(np.__version__) >= '0.16', +@pytest.mark.skipif('1.16.0' <= LooseVersion(np.__version__) < '1.16.1', reason='Until numpy/numpy#12745 is resolved') def test_serialize_deserialize_pandas(): # ARROW-1784, serialize and deserialize DataFrame by decomposing diff --git a/python/pyproject.toml b/python/pyproject.toml index 4eee45767d0..7c5b3d7c13b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -19,7 +19,9 @@ requires = [ "cython >= 0.29", "numpy==1.14.5; python_version<'3.7'", - "numpy==1.16.0; python_version>='3.7'", + "numpy==1.16.0; python_version=='3.7'", + "numpy==1.17.3; python_version=='3.8'", + "numpy==1.19.3; python_version>='3.9'", "setuptools", "setuptools_scm", "wheel" diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 67a98b03f3c..df25aeb3043 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,5 +1,7 @@ cython>=0.29.11 -numpy==1.14.5; python_version < "3.8" -numpy==1.17.3; python_version >= "3.8" +numpy==1.14.5; python_version < "3.7" +numpy==1.16.0; python_version == "3.7" +numpy==1.17.3; python_version == "3.8" +numpy==1.19.3; python_version >= "3.9" setuptools_scm wheel diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 07a494654cf..a327276eac6 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -2,8 +2,10 @@ cffi cython hypothesis==5.0; python_version <= "3.5.2" hypothesis; python_version > "3.5.2" -numpy==1.14.5; python_version < "3.8" -numpy==1.17.3; python_version >= "3.8" +numpy==1.14.5; python_version < "3.7" +numpy==1.16.0; python_version == "3.7" +numpy==1.17.3; python_version == "3.8" +numpy==1.19.3; python_version >= "3.9" pandas==0.24; python_version <= "3.5.2" pandas<1.1.0; python_version > "3.5.2" and python_version < "3.8" pandas; python_version >= "3.8" diff --git a/python/setup.py b/python/setup.py index d803f7684d5..0ab85955a1d 100755 --- a/python/setup.py +++ b/python/setup.py @@ -628,6 +628,7 @@ def has_ext_modules(foo): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], license='Apache License, Version 2.0', maintainer='Apache Arrow Developers',