From b89787c33ea5472097bbe9ba51c0e1509394b8e4 Mon Sep 17 00:00:00 2001 From: "Terence D. Honles" Date: Wed, 7 Oct 2020 14:36:00 -0700 Subject: [PATCH 01/12] ARROW-10224: [Python][Packaging] Build, test, and support Python 3.9 Updates vendored arrow-cpp-feedstock to abbcc009d4171a8d6c --- cpp/Brewfile | 1 - cpp/cmake_modules/ThirdpartyToolchain.cmake | 2 + dev/release/verify-release-candidate.sh | 4 +- .../linux_aarch64_python3.6.____cpython.yaml | 17 ++- .../linux_aarch64_python3.7.____cpython.yaml | 17 ++- .../linux_aarch64_python3.8.____cpython.yaml | 17 ++- .../linux_aarch64_python3.9.____cpython.yaml | 71 +++++++++++ ...piler_version9.2python3.6.____cpython.yaml | 12 +- ...piler_version9.2python3.7.____cpython.yaml | 12 +- ...piler_version9.2python3.8.____cpython.yaml | 12 +- ...piler_version9.2python3.9.____cpython.yaml | 69 +++++++++++ ...iler_versionNonepython3.6.____cpython.yaml | 12 +- ...iler_versionNonepython3.7.____cpython.yaml | 12 +- ...iler_versionNonepython3.8.____cpython.yaml | 12 +- ...iler_versionNonepython3.9.____cpython.yaml | 69 +++++++++++ .../osx_python3.6.____cpython.yaml | 13 +- .../osx_python3.7.____cpython.yaml | 15 ++- .../osx_python3.8.____cpython.yaml | 2 +- .../osx_python3.9.____cpython.yaml | 67 ++++++++++ .../win_python3.6.____cpython.yaml | 9 +- .../win_python3.7.____cpython.yaml | 9 +- .../win_python3.8.____cpython.yaml | 9 +- .../win_python3.9.____cpython.yaml | 55 ++++++++ dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 +- dev/tasks/docker-tests/github.linux.yml | 2 +- dev/tasks/python-wheels/osx-build.sh | 5 +- dev/tasks/python-wheels/travis.osx.yml | 17 ++- dev/tasks/python-wheels/win-build.bat | 2 +- dev/tasks/tasks.yml | 117 ++++++++++++++++++ docker-compose.yml | 2 +- python/manylinux1/Dockerfile-x86_64_base | 4 +- python/manylinux1/Dockerfile-x86_64_ubuntu | 2 +- python/manylinux1/build_arrow.sh | 2 +- .../manylinux201x/Dockerfile-x86_64_base_2010 | 4 +- .../manylinux201x/Dockerfile-x86_64_base_2014 | 4 +- python/manylinux201x/build_arrow.sh | 2 +- python/manylinux201x/scripts/install_cmake.sh | 4 +- python/pyarrow/tests/test_pandas.py | 4 +- python/pyproject.toml | 4 +- python/requirements-wheel-build.txt | 6 +- python/requirements-wheel-test.txt | 6 +- python/setup.py | 1 + 42 files changed, 621 insertions(+), 89 deletions(-) create mode 100644 dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml diff --git a/cpp/Brewfile b/cpp/Brewfile index 7de6c7deabe..2b7026b00a0 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -25,7 +25,6 @@ brew "flatbuffers" brew "git" brew "glog" brew "grpc" -brew "llvm" brew "llvm@8" brew "lz4" brew "minio" diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index d084f346803..563c62f4d63 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -768,6 +768,8 @@ if(MSVC AND ARROW_USE_STATIC_CRT) set(Boost_USE_STATIC_RUNTIME ON) endif() set(Boost_ADDITIONAL_VERSIONS + "1.74.0" + "1.74" "1.73.0" "1.73" "1.72.0" diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 0cb9e1b8b11..9a903e91bf2 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -634,7 +634,7 @@ IMPORT_TESTS } test_linux_wheels() { - local py_arches="3.5m 3.6m 3.7m 3.8" + local py_arches="3.5m 3.6m 3.7m 3.8 3.9" local manylinuxes="1 2010 2014" for py_arch in ${py_arches}; do @@ -659,7 +659,7 @@ test_linux_wheels() { } test_macos_wheels() { - local py_arches="3.5m 3.6m 3.7m 3.8" + local py_arches="3.5m 3.6m 3.7m 3.8 3.9" for py_arch in ${py_arches}; do local env=_verify_wheel-${py_arch} diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml index 47122636a58..f2d3ceaac68 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.6.____cpython.yaml @@ -1,7 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -9,7 +9,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '7.5' +- '7' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7.5' +- '7' docker_image: - condaforge/linux-anvil-aarch64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -39,7 +39,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -55,11 +55,16 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml index 1ce4521a92a..611c39c907c 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.7.____cpython.yaml @@ -1,7 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -9,7 +9,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '7.5' +- '7' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7.5' +- '7' docker_image: - condaforge/linux-anvil-aarch64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -39,7 +39,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -55,11 +55,16 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml index 9bf11fd5215..2f0fc0e2306 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.8.____cpython.yaml @@ -1,7 +1,7 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -9,7 +9,7 @@ bzip2: c_compiler: - gcc c_compiler_version: -- '7.5' +- '7' cdt_arch: - aarch64 cdt_name: @@ -23,7 +23,7 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7.5' +- '7' docker_image: - condaforge/linux-anvil-aarch64 gflags: @@ -31,7 +31,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -39,7 +39,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -55,11 +55,16 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml new file mode 100644 index 00000000000..2ec87205a0f --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_python3.9.____cpython.yaml @@ -0,0 +1,71 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '7' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-aarch64 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- linux-aarch64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml index 05234c57e34..4c5061d6ff2 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml index e4ab58abee6..db820f3ff26 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml index e15e3bf7623..24810f3c0f1 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml new file mode 100644 index 00000000000..ebe422be367 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml @@ -0,0 +1,69 @@ +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '9.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-cuda:9.2 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - docker_image +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml index e2952d8375d..358814fb0da 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml index 0da1784bdd6..054a730ee50 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml index 8fc3382d8a9..a3bfc58ea8f 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -27,7 +27,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ lz4_c: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,14 +51,18 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - linux-64 zip_keys: +- - c_compiler_version + - cxx_compiler_version - - cuda_compiler_version - docker_image +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml new file mode 100644 index 00000000000..88541395052 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml @@ -0,0 +1,69 @@ +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '7' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '7' +docker_image: +- condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - docker_image +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml index e4d163f526c..c019508dce4 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml @@ -1,7 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -25,7 +25,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ macos_machine: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -51,11 +51,16 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml index 3a805335574..8f66d609797 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml @@ -1,7 +1,7 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -25,7 +25,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -35,7 +35,7 @@ macos_machine: numpy: - '1.16' orc: -- 1.6.4 +- 1.6.5 pin_run_as_build: boost-cpp: max_pin: x.x.x @@ -49,13 +49,18 @@ pin_run_as_build: zlib: max_pin: x.x python: -- 3.7.* *_cpython +- 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml index 9e7aa65b8df..ad449921253 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml @@ -33,7 +33,7 @@ lz4_c: macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.16' +- '1.17.3' orc: - 1.6.4 pin_run_as_build: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml new file mode 100644 index 00000000000..c630217d111 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml @@ -0,0 +1,67 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '10' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '10' +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +macos_machine: +- x86_64-apple-darwin13.4.0 +numpy: +- '1.19' +orc: +- 1.6.5 +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml index bce682a27af..40199c679de 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -19,7 +19,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -41,11 +41,14 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - win-64 +zip_keys: +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml index 8b74b5d79a7..88d17108a98 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -19,7 +19,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -41,11 +41,14 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - win-64 +zip_keys: +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml index 32cbf60e952..e18785de135 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.8.54 +- 1.8.63 boost_cpp: - 1.74.0 bzip2: @@ -19,7 +19,7 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.32' libprotobuf: - '3.13' lz4_c: @@ -41,11 +41,14 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2020.10.01 snappy: - '1' target_platform: - win-64 +zip_keys: +- - numpy + - python zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml new file mode 100644 index 00000000000..6177f96ce01 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml @@ -0,0 +1,55 @@ +aws_sdk_cpp: +- 1.8.63 +boost_cpp: +- 1.74.0 +bzip2: +- '1' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- vs2017 +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.32' +libprotobuf: +- '3.13' +lz4_c: +- 1.9.2 +numpy: +- '1.19' +pin_run_as_build: + boost-cpp: + max_pin: x.x.x + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2020.10.01 +snappy: +- '1' +target_platform: +- win-64 +zip_keys: +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 272ed8c5382..cdbfc5d5b4d 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -179,7 +179,7 @@ outputs: host: - {{ pin_subpackage('arrow-cpp', exact=True) }} - cython - - numpy 1.16.* + - numpy - python - setuptools - setuptools_scm @@ -244,7 +244,7 @@ outputs: - {{ pin_subpackage('arrow-cpp', exact=True) }} - {{ pin_subpackage('pyarrow', exact=True) }} - cython - - numpy 1.16.* + - numpy - python - setuptools - setuptools_scm diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml index 1bc9e0742d7..c5851ed73ef 100644 --- a/dev/tasks/docker-tests/github.linux.yml +++ b/dev/tasks/docker-tests/github.linux.yml @@ -42,7 +42,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.9 - name: Setup Archery run: pip install -e arrow/dev/archery[docker] - name: Execute Docker Build diff --git a/dev/tasks/python-wheels/osx-build.sh b/dev/tasks/python-wheels/osx-build.sh index e029932ecf6..e6bd6159de9 100755 --- a/dev/tasks/python-wheels/osx-build.sh +++ b/dev/tasks/python-wheels/osx-build.sh @@ -50,7 +50,10 @@ function build_wheel { export ARROW_HOME=`pwd`/arrow-dist export PARQUET_HOME=`pwd`/arrow-dist - pip install $(pip_opts) -r python/requirements-wheel-build.txt + # If NumPy builds from the source make sure it is built against OpenBLAS + # See: https://github.com/numpy/numpy/issues/15947#issuecomment-686159427 + OPENBLAS=$(brew --prefix openblas) pip install \ + $(pip_opts) -r python/requirements-wheel-build.txt git submodule update --init export ARROW_TEST_DATA=`pwd`/testing/data diff --git a/dev/tasks/python-wheels/travis.osx.yml b/dev/tasks/python-wheels/travis.osx.yml index 99c667a6450..a67f36443c7 100644 --- a/dev/tasks/python-wheels/travis.osx.yml +++ b/dev/tasks/python-wheels/travis.osx.yml @@ -29,6 +29,7 @@ addons: - flatbuffers - flex - git + - openblas - openssl@1.1 - protobuf - python@3.8 @@ -70,7 +71,15 @@ install: # test the built wheels, move Homebrew directory to ensure # things are properly statically-linked - - sudo mv $(brew --cellar){,.bak} + - | + CELLAR=$(brew --cellar) + sudo mv $CELLAR{,.bak} + + # make a new Cellar, allowing numpy to dynamically link to openblas + sudo mkdir $CELLAR + sudo ln -s ../Cellar.bak/openblas $CELLAR/openblas + # and brew's openblas will need gcc + sudo ln -s ../Cellar.bak/gcc $CELLAR/gcc # install the built wheel and test dependencies - install_wheel arrow # run unit tests before removing the system libraries @@ -78,7 +87,11 @@ install: # run the import tests - run_import_tests - - sudo mv $(brew --cellar){.bak,} + - | + CELLAR=$(brew --cellar) + sudo rm $CELLAR/{gcc,openblas} + sudo rmdir $CELLAR + sudo mv $CELLAR{.bak,} # before_install activates a virtualenv but we need the system python3 - deactivate diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 9d02e787f0f..2dd3dd65ae0 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -114,6 +114,6 @@ python -c "import pyarrow.flight" || exit /B python -c "import pyarrow.dataset" || exit /B @rem run the python tests, but disable the cython because there is a linking -@rem issue on python 3.8 +@rem issue on python 3.9 set PYARROW_TEST_CYTHON=OFF python -m pytest -rs --pyargs pyarrow || exit /B diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index a2349f5b5a3..fa433b83108 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -174,6 +174,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-linux-gcc-py39-cpu: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_cuda_compiler_versionNonepython3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-linux-gcc-py36-cuda: ci: azure template: conda-recipes/azure.linux.yml @@ -201,6 +210,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 + conda-linux-gcc-py39-cuda: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_cuda_compiler_version9.2python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 + conda-linux-gcc-py36-aarch64: ci: drone template: conda-recipes/drone.yml @@ -228,6 +246,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-linux-gcc-py39-aarch64: + ci: drone + template: conda-recipes/drone.yml + params: + config: linux_aarch64_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Conda OSX #################################### conda-osx-clang-py36: @@ -259,6 +286,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-osx-clang-py39: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Conda Windows ################################ conda-win-vs2017-py36: @@ -290,6 +326,15 @@ tasks: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + conda-win-vs2017-py39: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + ############################## Wheel Linux ################################## wheel-manylinux1-cp35m: @@ -348,6 +393,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-manylinux1_x86_64.whl + wheel-manylinux1-cp39: + ci: azure + template: python-wheels/azure.linux.yml + params: + python_version: 3.9 + unicode_width: 16 + wheel_tag: manylinux1 + wheel_dir: manylinux1 + test_docker_images: + - python:3.9 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-manylinux1_x86_64.whl + wheel-manylinux2010-cp35m: ci: azure template: python-wheels/azure.linux.yml @@ -404,6 +463,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-manylinux2010_x86_64.whl + wheel-manylinux2010-cp39: + ci: azure + template: python-wheels/azure.linux.yml + params: + python_version: 3.9 + unicode_width: 16 + wheel_tag: manylinux2010 + wheel_dir: manylinux201x + test_docker_images: + - python:3.9 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-manylinux2010_x86_64.whl + wheel-manylinux2014-cp35m: ci: azure template: python-wheels/azure.linux.yml @@ -460,6 +533,20 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-manylinux2014_x86_64.whl + wheel-manylinux2014-cp39: + ci: azure + template: python-wheels/azure.linux.yml + params: + python_version: 3.9 + unicode_width: 16 + wheel_tag: manylinux2014 + wheel_dir: manylinux201x + test_docker_images: + - python:3.9 + test_remove_system_libs: true + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-manylinux2014_x86_64.whl + ############################## Wheel OSX #################################### wheel-osx-mavericks-cp35m: @@ -502,6 +589,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_9_x86_64.whl + wheel-osx-mavericks-cp39: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.9 + macos_deployment_target: 10.9 + arrow_s3: "OFF" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_9_x86_64.whl + # enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl wheel-osx-high-sierra-cp35m: @@ -544,6 +641,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-macosx_10_13_x86_64.whl + wheel-osx-high-sierra-cp39: + ci: travis + template: python-wheels/travis.osx.yml + params: + python_version: 3.9 + macos_deployment_target: 10.13 + arrow_s3: "ON" + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-macosx_10_13_x86_64.whl + ############################## Wheel Windows ################################ wheel-win-cp36m: @@ -570,6 +677,16 @@ tasks: artifacts: - pyarrow-{no_rc_version}-cp38-cp38-win_amd64.whl + wheel-win-cp39: + ci: appveyor + template: python-wheels/appveyor.yml + params: + script: win-build.bat + python_version: 3.9 + python_interpreter: C:\Python38-x64\python.exe + artifacts: + - pyarrow-{no_rc_version}-cp39-cp39-win_amd64.whl + ############################## Linux PKGS #################################### debian-stretch-amd64: diff --git a/docker-compose.yml b/docker-compose.yml index 195fc0104d2..4424943300b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -473,7 +473,7 @@ services: # docker-compose run --rm conda-python # Parameters: # ARCH: amd64, arm32v7 - # PYTHON: 3.6, 3.7, 3.8 + # PYTHON: 3.6, 3.7, 3.8, 3.9 image: ${REPO}:${ARCH}-conda-python-${PYTHON} build: context: . diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base index 15487e02fbc..8bc9cd2eb0f 100644 --- a/python/manylinux1/Dockerfile-x86_64_base +++ b/python/manylinux1/Dockerfile-x86_64_base @@ -17,7 +17,7 @@ # See https://quay.io/repository/pypa/manylinux1_x86_64?tab=tags # to update base image. -FROM quay.io/pypa/manylinux1_x86_64:2020-03-07-9c5ba95 +FROM quay.io/pypa/manylinux1_x86_64:2020-10-06-c4a4a7c # Install dependencies RUN yum install -y xz ccache flex && yum clean all @@ -89,7 +89,7 @@ RUN /build_glog.sh ENV GLOG_HOME /usr/local WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 # Remove unneeded Python versions RUN rm -rf /opt/_internal/cpython-2.7.* diff --git a/python/manylinux1/Dockerfile-x86_64_ubuntu b/python/manylinux1/Dockerfile-x86_64_ubuntu index b038f32af2a..290fbc1a68b 100644 --- a/python/manylinux1/Dockerfile-x86_64_ubuntu +++ b/python/manylinux1/Dockerfile-x86_64_ubuntu @@ -34,7 +34,7 @@ ADD scripts/install_cmake.sh / RUN /install_cmake.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 ADD scripts/build_openssl.sh / RUN /build_openssl.sh diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index ab67fced573..0bdd2e13ecf 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -62,7 +62,7 @@ export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost - mkdir -p /io/dist # Must pass PYTHON_VERSION env variable -# possible values are: 3.5 3.6 3.7 3.8 +# possible values are: 3.5 3.6 3.7 3.8 3.9 UNICODE_WIDTH=32 # Dummy value, irrelevant for Python 3 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" diff --git a/python/manylinux201x/Dockerfile-x86_64_base_2010 b/python/manylinux201x/Dockerfile-x86_64_base_2010 index 0d7d497ef5e..99883b711bd 100644 --- a/python/manylinux201x/Dockerfile-x86_64_base_2010 +++ b/python/manylinux201x/Dockerfile-x86_64_base_2010 @@ -17,7 +17,7 @@ # See https://quay.io/repository/pypa/manylinux2010_x86_64?tab=tags # to update base image. -FROM quay.io/pypa/manylinux2010_x86_64:2020-03-07-1825e8f +FROM quay.io/pypa/manylinux2010_x86_64:2020-10-06-786a931 # Install build dependencies RUN yum install -y xz bison ccache flex wget @@ -32,7 +32,7 @@ ADD scripts/build_zlib.sh / RUN /build_zlib.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 # Remove unneeded Python versions RUN rm -rf /opt/_internal/cpython-2.7.* diff --git a/python/manylinux201x/Dockerfile-x86_64_base_2014 b/python/manylinux201x/Dockerfile-x86_64_base_2014 index d8c71ad9c7f..93bafc40f8b 100644 --- a/python/manylinux201x/Dockerfile-x86_64_base_2014 +++ b/python/manylinux201x/Dockerfile-x86_64_base_2014 @@ -17,7 +17,7 @@ # See https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags # to update base image. -FROM quay.io/pypa/manylinux2014_x86_64:2020-03-07-06139da +FROM quay.io/pypa/manylinux2014_x86_64:2020-10-06-b2ca7a1 # Install build dependencies RUN yum install -y xz bison ccache flex wget @@ -32,7 +32,7 @@ ADD scripts/build_zlib.sh / RUN /build_zlib.sh WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 68a4af043e2adb0d9353d4a0e1f3d871203237aa +RUN git clone https://github.com/matthew-brett/multibuild.git && cd multibuild && git checkout 8882150df6529658700b66bec124dfb77eefca26 # Remove unneeded Python versions RUN rm -rf /opt/_internal/cpython-2.7.* diff --git a/python/manylinux201x/build_arrow.sh b/python/manylinux201x/build_arrow.sh index 3913b46b680..41586737991 100755 --- a/python/manylinux201x/build_arrow.sh +++ b/python/manylinux201x/build_arrow.sh @@ -62,7 +62,7 @@ export PKG_CONFIG_PATH=/usr/lib/pkgconfig:/arrow-dist/lib/pkgconfig mkdir -p /io/dist # Must pass PYTHON_VERSION env variable -# possible values are: 3.5 3.6 3.7 3.8 +# possible values are: 3.5 3.6 3.7 3.8 3.9 UNICODE_WIDTH=32 # Dummy value, irrelevant for Python 3 CPYTHON_PATH="$(cpython_path ${PYTHON_VERSION} ${UNICODE_WIDTH})" diff --git a/python/manylinux201x/scripts/install_cmake.sh b/python/manylinux201x/scripts/install_cmake.sh index 2afef8e70c9..474b72ecc75 100755 --- a/python/manylinux201x/scripts/install_cmake.sh +++ b/python/manylinux201x/scripts/install_cmake.sh @@ -18,6 +18,6 @@ export CMAKE_VERSION=3.18.2.post1 /opt/python/cp37-cp37m/bin/pip install cmake==${CMAKE_VERSION} ninja -ln -s /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake -ln -s /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja +ln -fs /opt/python/cp37-cp37m/bin/cmake /usr/local/bin/cmake +ln -fs /opt/python/cp37-cp37m/bin/ninja /usr/local/bin/ninja strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/* diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index f46e75fc3ac..44b62a95f40 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -1424,6 +1424,8 @@ def test_array_from_pandas_date_with_mask(self): expected = pd.Series([None, date(1991, 1, 1), None]) assert pa.Array.from_pandas(expected).equals(result) + @pytest.mark.skipif('1.16.0' <= LooseVersion(np.__version__) < '1.16.1', + reason='Until numpy/numpy#12745 is resolved') def test_fixed_offset_timezone(self): df = pd.DataFrame({ 'a': [ @@ -2787,7 +2789,7 @@ def _check_serialize_components_roundtrip(pd_obj): tm.assert_series_equal(pd_obj, deserialized) -@pytest.mark.skipif(LooseVersion(np.__version__) >= '0.16', +@pytest.mark.skipif('1.16.0' <= LooseVersion(np.__version__) < '1.16.1', reason='Until numpy/numpy#12745 is resolved') def test_serialize_deserialize_pandas(): # ARROW-1784, serialize and deserialize DataFrame by decomposing diff --git a/python/pyproject.toml b/python/pyproject.toml index 4eee45767d0..7c5b3d7c13b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -19,7 +19,9 @@ requires = [ "cython >= 0.29", "numpy==1.14.5; python_version<'3.7'", - "numpy==1.16.0; python_version>='3.7'", + "numpy==1.16.0; python_version=='3.7'", + "numpy==1.17.3; python_version=='3.8'", + "numpy==1.19.3; python_version>='3.9'", "setuptools", "setuptools_scm", "wheel" diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index 67a98b03f3c..df25aeb3043 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,5 +1,7 @@ cython>=0.29.11 -numpy==1.14.5; python_version < "3.8" -numpy==1.17.3; python_version >= "3.8" +numpy==1.14.5; python_version < "3.7" +numpy==1.16.0; python_version == "3.7" +numpy==1.17.3; python_version == "3.8" +numpy==1.19.3; python_version >= "3.9" setuptools_scm wheel diff --git a/python/requirements-wheel-test.txt b/python/requirements-wheel-test.txt index 07a494654cf..a327276eac6 100644 --- a/python/requirements-wheel-test.txt +++ b/python/requirements-wheel-test.txt @@ -2,8 +2,10 @@ cffi cython hypothesis==5.0; python_version <= "3.5.2" hypothesis; python_version > "3.5.2" -numpy==1.14.5; python_version < "3.8" -numpy==1.17.3; python_version >= "3.8" +numpy==1.14.5; python_version < "3.7" +numpy==1.16.0; python_version == "3.7" +numpy==1.17.3; python_version == "3.8" +numpy==1.19.3; python_version >= "3.9" pandas==0.24; python_version <= "3.5.2" pandas<1.1.0; python_version > "3.5.2" and python_version < "3.8" pandas; python_version >= "3.8" diff --git a/python/setup.py b/python/setup.py index d803f7684d5..0ab85955a1d 100755 --- a/python/setup.py +++ b/python/setup.py @@ -628,6 +628,7 @@ def has_ext_modules(foo): 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], license='Apache License, Version 2.0', maintainer='Apache Arrow Developers', From 3488e1c6fc6b6d5393daa26549a3ea023a627512 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Wed, 11 Nov 2020 06:07:13 +0900 Subject: [PATCH 02/12] Use GitHub --- dev/tasks/tasks.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index fa433b83108..603ebd82c48 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -678,12 +678,10 @@ tasks: - pyarrow-{no_rc_version}-cp38-cp38-win_amd64.whl wheel-win-cp39: - ci: appveyor - template: python-wheels/appveyor.yml + ci: github + template: python-wheels/github.win.yml params: - script: win-build.bat python_version: 3.9 - python_interpreter: C:\Python38-x64\python.exe artifacts: - pyarrow-{no_rc_version}-cp39-cp39-win_amd64.whl From 5e03e010bc65308452c95d51ed21f0baebebf059 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 02:40:01 +0900 Subject: [PATCH 03/12] Try using cmd and call to detect error code --- dev/tasks/python-wheels/github.win.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/github.win.yml b/dev/tasks/python-wheels/github.win.yml index f7638d8be36..17da17b73d7 100644 --- a/dev/tasks/python-wheels/github.win.yml +++ b/dev/tasks/python-wheels/github.win.yml @@ -53,9 +53,9 @@ jobs: run: | echo "c:\\Miniconda\\condabin" >> $GITHUB_PATH - name: Build wheel - shell: bash + shell: cmd run: | - arrow/dev/tasks/python-wheels/win-build.bat + call arrow/dev/tasks/python-wheels/win-build.bat - name: Prepare artifacts # the artifacts must be uploaded from a directory relative to the build root shell: bash From e4874a84fdb589391713c596e9c16c0ef2b168df Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 02:40:15 +0900 Subject: [PATCH 04/12] Remove vs2015_runtime pinning --- dev/tasks/python-wheels/win-build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 2dd3dd65ae0..453007d282e 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -26,7 +26,7 @@ call conda.bat create -n wheel-build -q -y -c conda-forge ^ --file=arrow\ci\conda_env_cpp.yml ^ --file=arrow\ci\conda_env_python.yml ^ - "vs2015_runtime<14.16" ^ + "vs2015_runtime" ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build From d9abeb78e20ab9a28a5180c2c5001851a53dfb4a Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 04:06:11 +0900 Subject: [PATCH 05/12] Use bundled Boost Because boost-cpp package in conda-forge sopped providing static library. --- dev/tasks/python-wheels/win-build.bat | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 453007d282e..966292ed585 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -26,7 +26,6 @@ call conda.bat create -n wheel-build -q -y -c conda-forge ^ --file=arrow\ci\conda_env_cpp.yml ^ --file=arrow\ci\conda_env_python.yml ^ - "vs2015_runtime" ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build @@ -44,23 +43,15 @@ pushd arrow\cpp\build @rem conda-forge also results in a broken build so we use the BUNDLED @rem dependency resolution strategy for now +@rem -DBoost_SOURCE=BUNDLED is required because boost-cpp package in +@rem conda-forge stopped providing static library. cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ - -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DARROW_BOOST_USE_SHARED=OFF ^ -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=OFF ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DARROW_DEPENDENCY_SOURCE=CONDA ^ - -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ -DARROW_CXXFLAGS="/MP" ^ - -DARROW_WITH_BZ2=OFF ^ - -DARROW_WITH_ZLIB=ON ^ - -DARROW_WITH_ZSTD=ON ^ - -DARROW_WITH_LZ4=ON ^ - -DARROW_WITH_SNAPPY=ON ^ - -DARROW_WITH_BROTLI=ON ^ -DARROW_DATASET=ON ^ + -DARROW_DEPENDENCY_SOURCE=CONDA ^ -DARROW_FLIGHT=ON ^ -DARROW_GANDIVA=OFF ^ -DARROW_LZ4_USE_SHARED=OFF ^ @@ -69,9 +60,19 @@ cmake -A "%ARCH%" ^ -DARROW_PYTHON=ON ^ -DARROW_SNAPPY_USE_SHARED=OFF ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=OFF ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + -DBoost_SOURCE=BUNDLED ^ -DBrotli_SOURCE=BUNDLED ^ - -Dzstd_SOURCE=BUNDLED ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ + -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ -Dutf8proc_SOURCE=BUNDLED ^ + -Dzstd_SOURCE=BUNDLED ^ .. || exit /B cmake ^ --build . ^ From 3806d8c58ce2158a266cfe54c3029d46d7b6b0c8 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 04:55:04 +0900 Subject: [PATCH 06/12] Fix a typo --- dev/tasks/python-wheels/win-build.bat | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 966292ed585..20f67e044d4 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -43,7 +43,7 @@ pushd arrow\cpp\build @rem conda-forge also results in a broken build so we use the BUNDLED @rem dependency resolution strategy for now -@rem -DBoost_SOURCE=BUNDLED is required because boost-cpp package in +@rem -DBOOST_SOURCE=BUNDLED is required because boost-cpp package in @rem conda-forge stopped providing static library. cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ @@ -66,7 +66,7 @@ cmake -A "%ARCH%" ^ -DARROW_WITH_SNAPPY=ON ^ -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ - -DBoost_SOURCE=BUNDLED ^ + -DBOOST_SOURCE=BUNDLED ^ -DBrotli_SOURCE=BUNDLED ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ From 63de89b280184ae5ca7f785b181b7a01634c1d06 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 06:21:14 +0900 Subject: [PATCH 07/12] Use ARROW_DEPENDENCY_SOURCE=BUNDLED --- dev/tasks/python-wheels/win-build.bat | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 20f67e044d4..cea702fb220 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -43,22 +43,21 @@ pushd arrow\cpp\build @rem conda-forge also results in a broken build so we use the BUNDLED @rem dependency resolution strategy for now -@rem -DBOOST_SOURCE=BUNDLED is required because boost-cpp package in -@rem conda-forge stopped providing static library. +@rem -DARROW_DEPENDENCY_SOURCE=BUNDLED is required because recent +@rem conda-forge packages don't provide static library. cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ -DARROW_BUILD_STATIC=OFF ^ -DARROW_BUILD_TESTS=OFF ^ -DARROW_CXXFLAGS="/MP" ^ -DARROW_DATASET=ON ^ - -DARROW_DEPENDENCY_SOURCE=CONDA ^ + -DARROW_DEPENDENCY_SOURCE=BUNDLED ^ -DARROW_FLIGHT=ON ^ -DARROW_GANDIVA=OFF ^ - -DARROW_LZ4_USE_SHARED=OFF ^ -DARROW_MIMALLOC=ON ^ + -DARROW_OPENSSL_USE_SHARED=OFF ^ -DARROW_PARQUET=ON ^ -DARROW_PYTHON=ON ^ - -DARROW_SNAPPY_USE_SHARED=OFF ^ -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DARROW_WITH_BROTLI=ON ^ -DARROW_WITH_BZ2=OFF ^ @@ -66,13 +65,9 @@ cmake -A "%ARCH%" ^ -DARROW_WITH_SNAPPY=ON ^ -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ - -DBOOST_SOURCE=BUNDLED ^ - -DBrotli_SOURCE=BUNDLED ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ - -Dutf8proc_SOURCE=BUNDLED ^ - -Dzstd_SOURCE=BUNDLED ^ .. || exit /B cmake ^ --build . ^ From 797da2423e1b82473af3140c2c6d8ac048ed7d7b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 09:10:37 +0900 Subject: [PATCH 08/12] Use libprotobuf-static --- dev/tasks/python-wheels/win-build.bat | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index cea702fb220..053b256de01 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -26,6 +26,7 @@ call conda.bat create -n wheel-build -q -y -c conda-forge ^ --file=arrow\ci\conda_env_cpp.yml ^ --file=arrow\ci\conda_env_python.yml ^ + libprotobuf-static ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build @@ -45,6 +46,9 @@ pushd arrow\cpp\build @rem -DARROW_DEPENDENCY_SOURCE=BUNDLED is required because recent @rem conda-forge packages don't provide static library. +@rem +@rem We can't use bundled protobuf (v3.12.1) because it causes build +@rem error. cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ -DARROW_BUILD_STATIC=OFF ^ @@ -56,9 +60,9 @@ cmake -A "%ARCH%" ^ -DARROW_GANDIVA=OFF ^ -DARROW_MIMALLOC=ON ^ -DARROW_OPENSSL_USE_SHARED=OFF ^ + -DARROW_PROTOBUF_USE_SHARED=OFF ^ -DARROW_PARQUET=ON ^ -DARROW_PYTHON=ON ^ - -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DARROW_WITH_BROTLI=ON ^ -DARROW_WITH_BZ2=OFF ^ -DARROW_WITH_LZ4=ON ^ @@ -68,6 +72,7 @@ cmake -A "%ARCH%" ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ + -DProtobuf_SOURCE=CONDA ^ .. || exit /B cmake ^ --build . ^ From 59da36b94b931ae1c72b7896f1aa847e904b011c Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 11:51:07 +0900 Subject: [PATCH 09/12] Use protobuf 3.13.0 --- cpp/thirdparty/versions.txt | 2 +- dev/tasks/python-wheels/win-build.bat | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 545943d30c6..7e494362794 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -41,7 +41,7 @@ ARROW_JEMALLOC_BUILD_VERSION=5.2.1 ARROW_LZ4_BUILD_VERSION=v1.9.2 ARROW_MIMALLOC_BUILD_VERSION=v1.6.4 ARROW_ORC_BUILD_VERSION=1.6.2 -ARROW_PROTOBUF_BUILD_VERSION=v3.12.1 +ARROW_PROTOBUF_BUILD_VERSION=v3.13.0 # Because of https://github.com/Tencent/rapidjson/pull/1323, we require # a pre-release version of RapidJSON to build with GCC 8 without # warnings. diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 053b256de01..122289062d8 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -26,7 +26,6 @@ call conda.bat create -n wheel-build -q -y -c conda-forge ^ --file=arrow\ci\conda_env_cpp.yml ^ --file=arrow\ci\conda_env_python.yml ^ - libprotobuf-static ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build @@ -46,9 +45,6 @@ pushd arrow\cpp\build @rem -DARROW_DEPENDENCY_SOURCE=BUNDLED is required because recent @rem conda-forge packages don't provide static library. -@rem -@rem We can't use bundled protobuf (v3.12.1) because it causes build -@rem error. cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ -DARROW_BUILD_STATIC=OFF ^ @@ -72,7 +68,6 @@ cmake -A "%ARCH%" ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ - -DProtobuf_SOURCE=CONDA ^ .. || exit /B cmake ^ --build . ^ From 5226bf79241b66866ed71e1f89699211843dc0dc Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 12:39:08 +0900 Subject: [PATCH 10/12] Use CMake style path --- dev/tasks/python-wheels/win-build.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 122289062d8..7ccabb1c1e0 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -45,6 +45,7 @@ pushd arrow\cpp\build @rem -DARROW_DEPENDENCY_SOURCE=BUNDLED is required because recent @rem conda-forge packages don't provide static library. +SET cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ -DARROW_BUILD_STATIC=OFF ^ @@ -67,7 +68,7 @@ cmake -A "%ARCH%" ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DOPENSSL_ROOT_DIR=%CONDA_PREFIX%/Library ^ + -DOPENSSL_ROOT_DIR=C:/Miniconda/envs/wheel-build/Library ^ .. || exit /B cmake ^ --build . ^ From fcb561bf960f23d3c1b85c791bb2acff8e2575a6 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Fri, 13 Nov 2020 14:34:39 +0900 Subject: [PATCH 11/12] Use gRPC 1.33.2 --- cpp/thirdparty/versions.txt | 2 +- dev/tasks/python-wheels/win-build.bat | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 7e494362794..62dc5077e0f 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -35,7 +35,7 @@ ARROW_CARES_BUILD_VERSION=1.16.1 ARROW_GBENCHMARK_BUILD_VERSION=v1.5.1 ARROW_GFLAGS_BUILD_VERSION=v2.2.2 ARROW_GLOG_BUILD_VERSION=v0.4.0 -ARROW_GRPC_BUILD_VERSION=v1.29.1 +ARROW_GRPC_BUILD_VERSION=v1.33.2 ARROW_GTEST_BUILD_VERSION=1.8.1 ARROW_JEMALLOC_BUILD_VERSION=5.2.1 ARROW_LZ4_BUILD_VERSION=v1.9.2 diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index 7ccabb1c1e0..ec06ee645c2 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -30,6 +30,7 @@ call conda.bat create -n wheel-build -q -y -c conda-forge ^ call conda.bat activate wheel-build +set CONDA_PREFIX_CMAKE_STYLE=%CONDA_PREFIX:\=/% set ARROW_HOME=%CONDA_PREFIX%\Library set PARQUET_HOME=%CONDA_PREFIX%\Library echo %ARROW_HOME% @@ -45,7 +46,6 @@ pushd arrow\cpp\build @rem -DARROW_DEPENDENCY_SOURCE=BUNDLED is required because recent @rem conda-forge packages don't provide static library. -SET cmake -A "%ARCH%" ^ -G "%GENERATOR%" ^ -DARROW_BUILD_STATIC=OFF ^ @@ -68,7 +68,7 @@ cmake -A "%ARCH%" ^ -DARROW_WITH_ZSTD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ - -DOPENSSL_ROOT_DIR=C:/Miniconda/envs/wheel-build/Library ^ + -DOPENSSL_ROOT_DIR=%CONDA_PREFIX_CMAKE_STYLE%/Library ^ .. || exit /B cmake ^ --build . ^ From ec8efef3fdb9dd2a5f9a651f1acd104ea484689c Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 26 Nov 2020 17:26:35 +0100 Subject: [PATCH 12/12] Build grpc --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 9 +++++++++ dev/tasks/python-wheels/win-build.bat | 9 +++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 563c62f4d63..a5d85f5263d 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1346,6 +1346,7 @@ macro(build_protobuf) -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_LIBDIR=lib "-DCMAKE_INSTALL_PREFIX=${PROTOBUF_PREFIX}" + -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_DEBUG_POSTFIX=) if(ZLIB_ROOT) @@ -2360,6 +2361,8 @@ macro(build_grpc) if(GFLAGS_VENDORED) add_dependencies(grpc_dependencies gflags_ep) endif() + resolve_dependency(RE2) + add_dependencies(grpc_dependencies re2_ep) add_dependencies(grpc_dependencies ${ARROW_PROTOBUF_LIBPROTOBUF} c-ares::cares ZLIB::ZLIB) @@ -2409,6 +2412,12 @@ macro(build_grpc) -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_SSL_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package + -DgRPC_PROTOBUF_PACKAGE_TYPE=CONFIG + -DgRPC_MSVC_STATIC_RUNTIME=OFF + -DgRPC_RE2_PROVIDER=package + -DRE2_ROOT_DIR=${RE2_PREFIX} + -Dre2_ROOT=${RE2_PREFIX} + -DCMAKE_POLICY_DEFAULT_CMP0074=NEW -DCMAKE_CXX_FLAGS=${GRPC_CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${EP_C_FLAGS} -DCMAKE_INSTALL_PREFIX=${GRPC_PREFIX} diff --git a/dev/tasks/python-wheels/win-build.bat b/dev/tasks/python-wheels/win-build.bat index ec06ee645c2..3e875fe3649 100644 --- a/dev/tasks/python-wheels/win-build.bat +++ b/dev/tasks/python-wheels/win-build.bat @@ -24,8 +24,7 @@ @rem create conda environment for compiling call conda.bat create -n wheel-build -q -y -c conda-forge ^ - --file=arrow\ci\conda_env_cpp.yml ^ - --file=arrow\ci\conda_env_python.yml ^ + cmake ninja pkg-config numpy=1.19 ^ python=%PYTHON_VERSION% || exit /B call conda.bat activate wheel-build @@ -66,10 +65,16 @@ cmake -A "%ARCH%" ^ -DARROW_WITH_SNAPPY=ON ^ -DARROW_WITH_ZLIB=ON ^ -DARROW_WITH_ZSTD=ON ^ + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DOPENSSL_ROOT_DIR=%CONDA_PREFIX_CMAKE_STYLE%/Library ^ .. || exit /B +cmake ^ + --build . ^ + --config Release ^ + --parallel %NUMBER_OF_PROCESSORS% ^ + --target grpc_ep || exit /B cmake ^ --build . ^ --config Release ^