From db678404a3134d674a7ea30128ac7f859b5581c0 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 21 Mar 2023 16:22:38 +1100 Subject: [PATCH 01/13] update arrow feedstock dependencies --- ...> linux_64_cuda_compiler_version10.2.yaml} | 18 ++++++---- ...> linux_64_cuda_compiler_versionNone.yaml} | 18 ++++++---- ...ux_aarch64_cuda_compiler_versionNone.yaml} | 18 ++++++---- ...ux_ppc64le_cuda_compiler_versionNone.yaml} | 18 ++++++---- .../{osx_64_openssl3.yaml => osx_64_.yaml} | 16 +++++---- ...sx_arm64_openssl3.yaml => osx_arm64_.yaml} | 16 +++++---- ... => win_64_cuda_compiler_version10.2.yaml} | 14 ++++---- ... => win_64_cuda_compiler_versionNone.yaml} | 14 ++++---- dev/tasks/tasks.yml | 34 +++++++++---------- 9 files changed, 95 insertions(+), 71 deletions(-) rename dev/tasks/conda-recipes/.ci_support/{linux_64_cuda_compiler_version10.2openssl3.yaml => linux_64_cuda_compiler_version10.2.yaml} (89%) rename dev/tasks/conda-recipes/.ci_support/{linux_64_cuda_compiler_versionNoneopenssl3.yaml => linux_64_cuda_compiler_versionNone.yaml} (89%) rename dev/tasks/conda-recipes/.ci_support/{linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml => linux_aarch64_cuda_compiler_versionNone.yaml} (89%) rename dev/tasks/conda-recipes/.ci_support/{linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml => linux_ppc64le_cuda_compiler_versionNone.yaml} (88%) rename dev/tasks/conda-recipes/.ci_support/{osx_64_openssl3.yaml => osx_64_.yaml} (89%) rename dev/tasks/conda-recipes/.ci_support/{osx_arm64_openssl3.yaml => osx_arm64_.yaml} (89%) rename dev/tasks/conda-recipes/.ci_support/{win_64_cuda_compiler_version10.2openssl3.yaml => win_64_cuda_compiler_version10.2.yaml} (90%) rename dev/tasks/conda-recipes/.ci_support/{win_64_cuda_compiler_versionNoneopenssl3.yaml => win_64_cuda_compiler_versionNone.yaml} (90%) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml index 31c61b17fc0..5d80a17c4df 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2.yaml @@ -1,5 +1,7 @@ +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -31,11 +33,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -48,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -59,13 +61,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-64 thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index e0629f86278..39b25b44690 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -1,5 +1,7 @@ +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -31,11 +33,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -48,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -59,13 +61,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-64 thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index 793c1afa7ab..af0fc2dcd25 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -1,7 +1,9 @@ BUILD: - aarch64-conda_cos7-linux-gnu +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -31,11 +33,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -48,7 +50,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -59,13 +61,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-aarch64 thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml similarity index 88% rename from dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index 6c81ae03b6d..83a1f7f7400 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -1,5 +1,7 @@ +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -27,11 +29,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -44,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -55,13 +57,15 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - linux-ppc64le thrift_cpp: -- 0.16.0 +- 0.18.1 +ucx: +- 1.14.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index bff4181d549..0cf990cc113 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,7 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -25,11 +27,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -44,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -55,13 +57,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - osx-64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml similarity index 89% rename from dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index 2e9a5682b95..3faa6278e81 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -1,7 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' +aws_crt_cpp: +- 0.19.8 aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -25,11 +27,11 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -44,7 +46,7 @@ numpy: openssl: - '3' orc: -- 1.8.0 +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -55,13 +57,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - osx-arm64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml similarity index 90% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml index 7173299488e..6ea00e3bd0d 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2openssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version10.2.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -23,15 +23,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libcrc32c: - '1.1' libcurl: - '7' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -43,6 +43,8 @@ numpy: - '1.20' openssl: - '3' +orc: +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -53,13 +55,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - win-64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - python - numpy diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml similarity index 90% rename from dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index 49fc4defc1b..183356662c6 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNoneopenssl3.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,5 +1,5 @@ aws_sdk_cpp: -- 1.9.379 +- 1.10.57 bzip2: - '1' c_ares: @@ -23,15 +23,15 @@ gflags: glog: - '0.6' google_cloud_cpp: -- 2.5.0 +- 2.8.0 libabseil: -- '20220623.0' +- '20230125' libcrc32c: - '1.1' libcurl: - '7' libgrpc: -- '1.51' +- '1.52' libprotobuf: - '3.21' lz4_c: @@ -43,6 +43,8 @@ numpy: - '1.20' openssl: - '3' +orc: +- 1.8.3 pin_run_as_build: python: min_pin: x.x @@ -53,13 +55,13 @@ python: - 3.8.* *_cpython - 3.9.* *_cpython re2: -- 2022.06.01 +- 2023.02.02 snappy: - '1' target_platform: - win-64 thrift_cpp: -- 0.16.0 +- 0.18.1 zip_keys: - - python - numpy diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 03dbe96deb0..f9f074e3690 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -250,7 +250,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_versionNoneopenssl3 + config: linux_64_cuda_compiler_versionNone r_config: linux_64_r_base4.1 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -260,7 +260,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_versionNoneopenssl3 + config: linux_64_cuda_compiler_versionNone r_config: linux_64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -270,7 +270,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_versionNoneopenssl3 + config: linux_64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -282,7 +282,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_64_cuda_compiler_version10.2openssl3 + config: linux_64_cuda_compiler_version10.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda @@ -296,7 +296,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_aarch64_cuda_compiler_versionNoneopenssl3 + config: linux_aarch64_cuda_compiler_versionNone r_config: linux_aarch64_r_base4.1 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -306,7 +306,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_aarch64_cuda_compiler_versionNoneopenssl3 + config: linux_aarch64_cuda_compiler_versionNone r_config: linux_aarch64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -316,7 +316,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_aarch64_cuda_compiler_versionNoneopenssl3 + config: linux_aarch64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -330,7 +330,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_ppc64le_cuda_compiler_versionNoneopenssl3 + config: linux_ppc64le_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -344,7 +344,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_openssl3 + config: osx_64_ r_config: osx_64_r_base4.1 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -354,7 +354,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_openssl3 + config: osx_64_ r_config: osx_64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -364,7 +364,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_64_openssl3 + config: osx_64_ artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -378,7 +378,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_arm64_openssl3 + config: osx_arm64_ r_config: osx_arm64_r_base4.1 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -388,7 +388,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_arm64_openssl3 + config: osx_arm64_ r_config: osx_arm64_r_base4.2 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -398,7 +398,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_arm64_openssl3 + config: osx_arm64_ artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -412,7 +412,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNoneopenssl3 + config: win_64_cuda_compiler_versionNone r_config: win_64_ artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda @@ -424,7 +424,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNoneopenssl3 + config: win_64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.conda @@ -436,7 +436,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_64_cuda_compiler_versionNoneopenssl3 + config: win_64_cuda_compiler_versionNone artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cuda.conda - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.conda From 10f80fee2243604efac9522c11f36fdc8808d1b0 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 21 Mar 2023 16:31:31 +1100 Subject: [PATCH 02/13] sync with conda-forge recipe --- dev/tasks/conda-recipes/arrow-cpp/activate.sh | 30 ++++++ .../conda-recipes/arrow-cpp/bld-arrow.bat | 13 ++- .../conda-recipes/arrow-cpp/bld-pyarrow.bat | 3 +- .../conda-recipes/arrow-cpp/build-arrow.sh | 27 ++++- .../conda-recipes/arrow-cpp/build-pyarrow.sh | 4 +- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 101 +++++++++--------- 6 files changed, 120 insertions(+), 58 deletions(-) create mode 100644 dev/tasks/conda-recipes/arrow-cpp/activate.sh diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh b/dev/tasks/conda-recipes/arrow-cpp/activate.sh new file mode 100644 index 00000000000..90210fac0a0 --- /dev/null +++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# for the gdb-wrappers, we need to create a symlink that +# contains the full path of the lib _within_ the installed +# env, which we don't have until the env is created. + +# doesn't come with a deactivate script, because the symlink +# is benign and doesn't need to be deleted. + +# where the GDB wrappers get installed +GDB_PREFIX=$CONDA_PREFIX/share/gdb/auto-load + +# If the directory is not writable, nothing can be done +if [ ! -w $GDB_PREFIX ]; then + return +fi + +# this needs to be in sync with the respective patch +PLACEHOLDER=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX +# the paths here are intentionally stacked, see #935, resp. +# https://github.com/apache/arrow/blob/master/docs/source/cpp/gdb.rst#manual-loading +WRAPPER_DIR=$GDB_PREFIX/$CONDA_PREFIX/lib + +mkdir -p $WRAPPER_DIR +# there's only one lib in that folder, but the libname changes +# based on the version so use a loop instead of hardcoding it. +for f in $GDB_PREFIX/$PLACEHOLDER/lib/*.py; do + # overwrite, because we don't have deactivation (i.e. symlink remains) + ln -sf $f $WRAPPER_DIR/$(basename $f) +done diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat index b3c283ddb8f..60c81be7411 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat @@ -14,6 +14,11 @@ if "%cuda_compiler_version%"=="None" ( set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON" ) +:: # reusable variable for dependencies we cannot yet unvendor +set "READ_RECIPE_META_YAML_WHY_NOT=OFF" + +:: for available switches see +:: https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake cmake -G "Ninja" ^ -DARROW_BOOST_USE_SHARED:BOOL=ON ^ -DARROW_BUILD_STATIC:BOOL=OFF ^ @@ -32,7 +37,7 @@ cmake -G "Ninja" ^ -DARROW_HDFS:BOOL=ON ^ -DARROW_JSON:BOOL=ON ^ -DARROW_MIMALLOC:BOOL=ON ^ - -DARROW_ORC:BOOL=OFF ^ + -DARROW_ORC:BOOL=ON ^ -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^ -DARROW_PARQUET:BOOL=ON ^ -DARROW_S3:BOOL=ON ^ @@ -42,6 +47,8 @@ cmake -G "Ninja" ^ -DARROW_WITH_BROTLI:BOOL=ON ^ -DARROW_WITH_BZ2:BOOL=ON ^ -DARROW_WITH_LZ4:BOOL=ON ^ + -DARROW_WITH_NLOHMANN_JSON:BOOL=ON ^ + -DARROW_WITH_OPENTELEMETRY:BOOL=%READ_RECIPE_META_YAML_WHY_NOT% ^ -DARROW_WITH_SNAPPY:BOOL=ON ^ -DARROW_WITH_ZLIB:BOOL=ON ^ -DARROW_WITH_ZSTD:BOOL=ON ^ @@ -56,9 +63,9 @@ cmake -G "Ninja" ^ -DPython3_EXECUTABLE="%PYTHON%" ^ %EXTRA_CMAKE_ARGS% ^ .. -if errorlevel 1 exit 1 +if %ERRORLEVEL% neq 0 exit 1 cmake --build . --target install --config Release -if errorlevel 1 exit 1 +if %ERRORLEVEL% neq 0 exit 1 popd diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat index ed0e20554b7..084faf74e4a 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -1,4 +1,5 @@ @echo on + pushd "%SRC_DIR%"\python @rem the symlinks for cmake modules don't work here @@ -39,7 +40,7 @@ if "%cuda_compiler_version%"=="None" ( build_ext ^ install --single-version-externally-managed ^ --record=record.txt -if errorlevel 1 exit 1 +if %ERRORLEVEL% neq 0 exit 1 popd if [%PKG_NAME%] == [pyarrow] ( diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index ddeec5b75cf..1f47986f9ea 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -1,7 +1,13 @@ #!/bin/bash +set -ex -set -e -set -x +# Copy the [de]activate scripts to $PREFIX/etc/conda/[de]activate.d, see +# https://conda-forge.org/docs/maintainer/adding_pkgs.html#activate-scripts +for CHANGE in "activate" +do + mkdir -p "${PREFIX}/etc/conda/${CHANGE}.d" + cp "${RECIPE_DIR}/${CHANGE}.sh" "${PREFIX}/etc/conda/${CHANGE}.d/${PKG_NAME}_${CHANGE}.sh" +done mkdir cpp/build pushd cpp/build @@ -12,10 +18,13 @@ EXTRA_CMAKE_ARGS="" if [ "$(uname)" == "Linux" ]; then SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') ARROW_GANDIVA_PC_CXX_FLAGS="${SYSTEM_INCLUDES}" + # only available on linux + ARROW_WITH_UCX=ON else # See https://conda-forge.org/docs/maintainer/knowledge_base.html#newer-c-features-with-old-sdk CXXFLAGS="${CXXFLAGS} -D_LIBCPP_DISABLE_AVAILABILITY" ARROW_GANDIVA_PC_CXX_FLAGS="-D_LIBCPP_DISABLE_AVAILABILITY" + ARROW_WITH_UCX=OFF fi # Enable CUDA support @@ -55,6 +64,16 @@ if [[ "${target_platform}" == "linux-aarch64" ]] || [[ "${target_platform}" == " export CMAKE_BUILD_PARALLEL_LEVEL=3 fi +# point to a usable protoc if we're running on a different architecture than the target +if [[ "${build_platform}" != "${target_platform}" ]]; then + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc" +fi + +# reusable variable for dependencies we cannot yet unvendor +export READ_RECIPE_META_YAML_WHY_NOT=OFF + +# for available switches see +# https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake cmake -GNinja \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ @@ -88,7 +107,10 @@ cmake -GNinja \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_NLOHMANN_JSON=ON \ + -DARROW_WITH_OPENTELEMETRY=${READ_RECIPE_META_YAML_WHY_NOT} \ -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_UCX=${ARROW_WITH_UCX} \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ -DBUILD_SHARED_LIBS=ON \ @@ -98,7 +120,6 @@ cmake -GNinja \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ -DPARQUET_REQUIRE_ENCRYPTION=ON \ - -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc \ -DPython3_EXECUTABLE=${PYTHON} \ ${EXTRA_CMAKE_ARGS} \ .. diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 5d5d98d1c8d..14c67ede632 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -1,7 +1,5 @@ #!/bin/sh - -set -e -set -x +set -ex # Build dependencies export ARROW_HOME=$PREFIX diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 3aa30a65025..1033b14528f 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -6,6 +6,9 @@ {% set proc_build_number = "0" %} {% set llvm_version = "14" %} +# see https://github.com/apache/arrow/blob/apache-arrow-10.0.1/cpp/CMakeLists.txt#L88-L90 +{% set so_version = (version.split(".")[0] | int * 100 + version.split(".")[1] | int) ~ "." ~ version.split(".")[2] ~ ".0" %} + package: name: apache-arrow version: {{ version }} @@ -30,12 +33,6 @@ outputs: build: number: {{ proc_build_number }} string: {{ build_ext }} - ignore_run_exports_from: - - openssl - requirements: - host: - # ensure we split the CI jobs per openssl version - - openssl test: commands: - exit 0 @@ -52,12 +49,7 @@ outputs: build: number: {{ proc_build_number }} string: {{ build_ext }} - ignore_run_exports_from: - - openssl requirements: - host: - # ensure we split the CI jobs per openssl version - - openssl run: - apache-arrow-proc ={{ build_ext_version }}={{ build_ext }} test: @@ -80,29 +72,39 @@ outputs: - "*/nvcuda.dll" # [win] requirements: build: + - {{ compiler("c") }} + - {{ compiler("cxx") }} + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - clangdev {{ llvm_version }} # [osx and arm64] - llvmdev {{ llvm_version }} # [osx and arm64] - gnuconfig # [osx and arm64] - - libgrpc - - libprotobuf + # needs to run protoc & grpc_cpp_plugin + - libgrpc # [build_platform != target_platform] + - libprotobuf # [build_platform != target_platform] - cmake - - autoconf # [unix] - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + # necessary for vendored jemalloc + - autoconf # [linux] + - make # [linux] host: + # for required dependencies, see + # https://github.com/apache/arrow/blob/apache-arrow-11.0.0/cpp/cmake_modules/ThirdpartyToolchain.cmake#L46-L75 - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} + - aws-crt-cpp # [unix] - aws-sdk-cpp - boost-cpp >=1.70 - brotli - bzip2 - c-ares + # not yet: https://github.com/conda-forge/cpp-opentelemetry-sdk-feedstock/issues/38 + # - cpp-opentelemetry-sdk + # - proto-opentelemetry-proto =={{ cpp_opentelemetry_sdk }} - gflags - glog - google-cloud-cpp + # arrow uses a customized jemalloc, see #944 + # - jemalloc - libabseil # since libgoogle-cloud is static on windows, see # https://github.com/conda-forge/google-cloud-cpp-feedstock/pull/108, @@ -113,13 +115,15 @@ outputs: - libprotobuf - libutf8proc - lz4-c + - nlohmann_json # gandiva depends on openssl - openssl - - orc # [unix] + - orc - rapidjson - re2 - snappy - thrift-cpp + - ucx # [linux] - xsimd - zlib - zstd @@ -164,6 +168,10 @@ outputs: - if exist %LIBRARY_LIB%\arrow_cuda.lib exit 1 # [(cuda_compiler_version == "None") and win] - if exist %LIBRARY_LIB%\arrow_cuda_static.lib exit 1 # [(cuda_compiler_version == "None") and win] + # gdb-wrapper (paths are stacked intentionally) + - test -f $PREFIX/share/gdb/auto-load/$PREFIX/lib/libarrow.so.{{ so_version }}-gdb.py # [linux] + - test -f $PREFIX/share/gdb/auto-load/$PREFIX/lib/libarrow.{{ so_version }}.dylib-gdb.py # [osx] + about: home: http://github.com/apache/arrow license: Apache-2.0 @@ -180,8 +188,6 @@ outputs: - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} requirements: host: - # make sure the builds for different openssl versions (with the same PKG_BUILDNUM) have different hashes - - openssl - {{ pin_subpackage('libarrow', exact=True) }} run: - {{ pin_subpackage('libarrow', exact=True) }} @@ -197,30 +203,26 @@ outputs: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} ignore_run_exports: - cudatoolkit - ignore_run_exports_from: - - openssl track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} + rpaths: + - lib/ + - {{ SP_DIR }}/pyarrow missing_dso_whitelist: # not actually missing, but installed into SP_DIR, see tests - - "*/libarrow_python.so.*" # [linux] - - "*/libarrow_python_flight.so.*" # [linux] - - "*/libarrow_python.*.dylib" # [osx] - - "*/libarrow_python_flight.*.dylib" # [osx] - "*/arrow_python.dll" # [win] - "*/arrow_python_flight.dll" # [win] requirements: build: + - {{ compiler("c") }} + - {{ compiler("cxx") }} + # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - cython # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - {{ pin_subpackage('libarrow', exact=True) }} - clangdev {{ llvm_version }} @@ -228,7 +230,6 @@ outputs: - cython - gflags # [unix] - numpy - - openssl - python - setuptools - setuptools_scm @@ -294,22 +295,19 @@ outputs: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} ignore_run_exports: - cudatoolkit - ignore_run_exports_from: - - openssl track_features: {{ "[arrow-cuda]" if cuda_enabled else "" }} requirements: build: + - {{ compiler("c") }} + - {{ compiler("cxx") }} + # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] - python # [build_platform != target_platform] - cross-python_{{ target_platform }} # [build_platform != target_platform] - cython # [build_platform != target_platform] - numpy # [build_platform != target_platform] - cmake - ninja - - make # [unix] - - {{ compiler('c') }} - - {{ compiler('cxx') }} - # pyarrow does not require nvcc but it needs to link against libraries in libarrow=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - {{ pin_subpackage('libarrow', exact=True) }} - {{ pin_subpackage('pyarrow', exact=True) }} @@ -317,7 +315,6 @@ outputs: - llvmdev {{ llvm_version }} - cython - numpy - - openssl - python - setuptools - setuptools_scm @@ -340,17 +337,17 @@ outputs: - fsspec - hypothesis - pandas - # not all arches available in conda-forge; osx-arm not tested anyway - - pytorch * # [unix and not ppc64le and py<311] - scipy + # not all OSes/arches available in conda-forge + - pytorch * # [unix and not ppc64le] + # not yet rebuilt for libabseil 20230125 + # - tensorflow # [unix and x86_64 and py<311] # we're not building java bindings # - jpype1 # doesn't get picked up correctly # - libhdfs3 # causes segfaults # - sparse - # not rebuilt for grpc 1.49 yet - # - tensorflow source_files: - testing/data commands: @@ -372,6 +369,16 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv + # segfaults on OSX: to investigate ASAP + {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] + # failing on linux with "OSError: Could not connect to socket /tmp/[...]/plasma.sock" + {% set tests_to_skip = tests_to_skip + " or test_plasma" %} # [linux] + # gandiva tests are segfaulting on ppc + {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] + # test failures on ppc + {% set tests_to_skip = tests_to_skip + " or test_safe_cast_from_float_with_nans_to_int" %} # [ppc64le] + # gandiva tests are segfaulting on ppc + {% set tests_to_skip = tests_to_skip + " or test_float_with_null_as_integer" %} # [ppc64le] # "Unsupported backend 'nonexistent' specified in ARROW_DEFAULT_MEMORY_POOL" {% set tests_to_skip = tests_to_skip + " or (test_memory and test_env_var)" %} # [unix] # test is broken; header is in $PREFIX, not $SP_DIR @@ -379,9 +386,7 @@ outputs: # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough - - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or ((py==310) and not ppc64le)] - # NOTE: test suite is skipped on ppc due to bugs in QEMU code that cause - # CI to fail, even though the tests would run through on native hardware + - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or (py==310)] about: home: http://github.com/apache/arrow From a3e5c665016e94d763ecd20a88df8d792f72a67f Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Tue, 21 Mar 2023 19:13:39 +1100 Subject: [PATCH 03/13] add patch to make gdb-integration work with conda --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 +++ ...elocatable-CMAKE_INSTALL_FULL_LIBDIR.patch | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 1033b14528f..4a5eede3e90 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -15,6 +15,10 @@ package: source: path: ../../../../ + patches: + # patch that's necessary to make gdb integration work, due to the fact that + # we're not installing into the final environment when building with conda + - patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch build: number: 0 diff --git a/dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch b/dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch new file mode 100644 index 00000000000..b84564c406e --- /dev/null +++ b/dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch @@ -0,0 +1,26 @@ +From 7f1350d76bf71a8123cf78abe18ddb5876a7ca80 Mon Sep 17 00:00:00 2001 +From: "H. Vetinari" +Date: Thu, 26 Jan 2023 12:37:02 +1100 +Subject: [PATCH] don't bake non-relocatable CMAKE_INSTALL_FULL_LIBDIR into + gdb-integration + +--- + cpp/src/arrow/CMakeLists.txt | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt +index 90ab1e6ac..c1f51e0da 100644 +--- a/cpp/src/arrow/CMakeLists.txt ++++ b/cpp/src/arrow/CMakeLists.txt +@@ -644,7 +644,7 @@ if(ARROW_BUILD_SHARED AND NOT WIN32) + endif() + else() + set(ARROW_GDB_AUTO_LOAD_LIBARROW_GDB_DIR +- "${ARROW_GDB_AUTO_LOAD_DIR}/${CMAKE_INSTALL_FULL_LIBDIR}") ++ "${ARROW_GDB_AUTO_LOAD_DIR}/replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX/lib/") + set(ARROW_GDB_AUTO_LOAD_LIBARROW_GDB_INSTALL TRUE) + endif() + if(ARROW_GDB_AUTO_LOAD_LIBARROW_GDB_INSTALL) +-- +2.38.1.windows.1 + From d5a8673c939f2f4ef58bd6343a7d18d1da9747cd Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 22 Mar 2023 11:54:22 +1100 Subject: [PATCH 04/13] add compiler for a test that needs it --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 4a5eede3e90..af713c68352 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -331,6 +331,8 @@ outputs: test: requires: + # test_cpp_extension_in_python requires a compiler + - {{ compiler("c") }} # [unix] - pytest - pytest-lazy-fixture - backports.zoneinfo # [py<39] From 2b2556a015a3d1af4136e67c202c8e99be988647 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 6 Apr 2023 00:35:30 +0200 Subject: [PATCH 05/13] remove plasma test skip --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index af713c68352..2ffca8a3dcf 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -377,8 +377,6 @@ outputs: # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv # segfaults on OSX: to investigate ASAP {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] - # failing on linux with "OSError: Could not connect to socket /tmp/[...]/plasma.sock" - {% set tests_to_skip = tests_to_skip + " or test_plasma" %} # [linux] # gandiva tests are segfaulting on ppc {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] # test failures on ppc From fc35331ce5dd15983cda5a0955e8a61c695f4c61 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Apr 2023 10:22:34 +1100 Subject: [PATCH 06/13] replace patch with new gdb install option --- .../conda-recipes/arrow-cpp/build-arrow.sh | 2 ++ dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 --- ...elocatable-CMAKE_INSTALL_FULL_LIBDIR.patch | 26 ------------------- 3 files changed, 2 insertions(+), 30 deletions(-) delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 1f47986f9ea..989772fcaae 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -74,6 +74,7 @@ export READ_RECIPE_META_YAML_WHY_NOT=OFF # for available switches see # https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake +# ARROW_GDB_INSTALL_DIR must match what's used for replacement in activate.sh cmake -GNinja \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ @@ -92,6 +93,7 @@ cmake -GNinja \ -DARROW_GANDIVA=ON \ -DARROW_GANDIVA_PC_CXX_FLAGS="${ARROW_GANDIVA_PC_CXX_FLAGS}" \ -DARROW_GCS=ON \ + -DARROW_GDB_INSTALL_DIR=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_JSON=ON \ diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 2ffca8a3dcf..18311229e54 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -15,10 +15,6 @@ package: source: path: ../../../../ - patches: - # patch that's necessary to make gdb integration work, due to the fact that - # we're not installing into the final environment when building with conda - - patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch build: number: 0 diff --git a/dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch b/dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch deleted file mode 100644 index b84564c406e..00000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/patches/0001-don-t-bake-non-relocatable-CMAKE_INSTALL_FULL_LIBDIR.patch +++ /dev/null @@ -1,26 +0,0 @@ -From 7f1350d76bf71a8123cf78abe18ddb5876a7ca80 Mon Sep 17 00:00:00 2001 -From: "H. Vetinari" -Date: Thu, 26 Jan 2023 12:37:02 +1100 -Subject: [PATCH] don't bake non-relocatable CMAKE_INSTALL_FULL_LIBDIR into - gdb-integration - ---- - cpp/src/arrow/CMakeLists.txt | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt -index 90ab1e6ac..c1f51e0da 100644 ---- a/cpp/src/arrow/CMakeLists.txt -+++ b/cpp/src/arrow/CMakeLists.txt -@@ -644,7 +644,7 @@ if(ARROW_BUILD_SHARED AND NOT WIN32) - endif() - else() - set(ARROW_GDB_AUTO_LOAD_LIBARROW_GDB_DIR -- "${ARROW_GDB_AUTO_LOAD_DIR}/${CMAKE_INSTALL_FULL_LIBDIR}") -+ "${ARROW_GDB_AUTO_LOAD_DIR}/replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX/lib/") - set(ARROW_GDB_AUTO_LOAD_LIBARROW_GDB_INSTALL TRUE) - endif() - if(ARROW_GDB_AUTO_LOAD_LIBARROW_GDB_INSTALL) --- -2.38.1.windows.1 - From f365b3594086b4c328eb274cc0502a371b0a16b4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Apr 2023 11:10:26 +1100 Subject: [PATCH 07/13] sync back recipe from r-arrow feedstock --- dev/tasks/conda-recipes/r-arrow/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/tasks/conda-recipes/r-arrow/meta.yaml b/dev/tasks/conda-recipes/r-arrow/meta.yaml index 3d053a781b0..4c86dc9280c 100644 --- a/dev/tasks/conda-recipes/r-arrow/meta.yaml +++ b/dev/tasks/conda-recipes/r-arrow/meta.yaml @@ -10,6 +10,8 @@ source: path: ../../../../ build: + # 4.1 not usable anymore unless https://github.com/conda-forge/r-base-feedstock/pull/236 gets merged + skip: true # [unix and (r_base == "4.1")] merge_build_host: true # [win] number: 0 rpaths: From b2a22ade057fa3248b4ce25cba33c8be110d2488 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Apr 2023 11:16:38 +1100 Subject: [PATCH 08/13] adapt to now-skipped R 4.1 builds (sync CI from r-arrow) --- .../.ci_support/r/linux_64_r_base4.1.yaml | 27 ------------- .../r/linux_aarch64_r_base4.1.yaml | 31 -------------- .../.ci_support/r/osx_64_r_base4.1.yaml | 27 ------------- .../.ci_support/r/osx_arm64_r_base4.1.yaml | 27 ------------- dev/tasks/tasks.yml | 40 ------------------- 5 files changed, 152 deletions(-) delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml delete mode 100644 dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml deleted file mode 100644 index 96784cc8b6f..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -c_compiler: -- gcc -c_compiler_version: -- '11' -cdt_name: -- cos6 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '11' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- linux-64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml deleted file mode 100644 index 413d8c6583f..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml +++ /dev/null @@ -1,31 +0,0 @@ -BUILD: -- aarch64-conda_cos7-linux-gnu -c_compiler: -- gcc -c_compiler_version: -- '11' -cdt_arch: -- aarch64 -cdt_name: -- cos7 -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- gxx -cxx_compiler_version: -- '11' -docker_image: -- quay.io/condaforge/linux-anvil-cos7-x86_64 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- linux-aarch64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml deleted file mode 100644 index 7e5b3d08039..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '10.9' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -macos_machine: -- x86_64-apple-darwin13.4.0 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- osx-64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml deleted file mode 100644 index 5c58110332f..00000000000 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml +++ /dev/null @@ -1,27 +0,0 @@ -MACOSX_DEPLOYMENT_TARGET: -- '11.0' -c_compiler: -- clang -c_compiler_version: -- '14' -channel_sources: -- conda-forge -channel_targets: -- conda-forge main -cxx_compiler: -- clangxx -cxx_compiler_version: -- '14' -macos_machine: -- arm64-apple-darwin20.0.0 -pin_run_as_build: - r-base: - min_pin: x.x - max_pin: x.x -r_base: -- '4.1' -target_platform: -- osx-arm64 -zip_keys: -- - c_compiler_version - - cxx_compiler_version diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index f9f074e3690..06e2d19dbe8 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -246,16 +246,6 @@ tasks: # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. - conda-linux-x64-cpu-r41: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_64_cuda_compiler_versionNone - r_config: linux_64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-linux-x64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml @@ -292,16 +282,6 @@ tasks: ########################### Conda Linux (aarch64) ########################### - conda-linux-aarch64-cpu-r41: - ci: azure - template: conda-recipes/azure.linux.yml - params: - config: linux_aarch64_cuda_compiler_versionNone - r_config: linux_aarch64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-linux-aarch64-cpu-r42: ci: azure template: conda-recipes/azure.linux.yml @@ -340,16 +320,6 @@ tasks: ############################## Conda OSX (x64) ############################## - conda-osx-x64-cpu-r41: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_64_ - r_config: osx_64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-osx-x64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml @@ -374,16 +344,6 @@ tasks: ############################# Conda OSX (arm64) ############################# - conda-osx-arm64-cpu-r41: - ci: azure - template: conda-recipes/azure.osx.yml - params: - config: osx_arm64_ - r_config: osx_arm64_r_base4.1 - artifacts: - - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda - conda-osx-arm64-cpu-r42: ci: azure template: conda-recipes/azure.osx.yml From 0ea1fdb8e24fa5dc95d53b27c7e601e4f33f5669 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Apr 2023 11:47:11 +1100 Subject: [PATCH 09/13] match windows CI changes --- dev/tasks/conda-recipes/azure.win.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/azure.win.yml b/dev/tasks/conda-recipes/azure.win.yml index 95c5fc10056..ee4533a3ce7 100755 --- a/dev/tasks/conda-recipes/azure.win.yml +++ b/dev/tasks/conda-recipes/azure.win.yml @@ -3,7 +3,7 @@ jobs: - job: win pool: - vmImage: windows-2019 + vmImage: windows-2022 timeoutInMinutes: 360 variables: CONFIG: {{ config }} @@ -32,7 +32,7 @@ jobs: - script: | call activate base - mamba.exe install 'python=3.9' conda-build conda pip boa 'conda-forge-ci-setup=3' -c conda-forge --strict-channel-priority --yes + mamba.exe install 'python=3.10' conda-build conda pip boa 'conda-forge-ci-setup=3' -c conda-forge --strict-channel-priority --yes displayName: Install conda-build - script: set PYTHONUNBUFFERED=1 From ba2f4dbb54abfe1b9ec1d2da1c40038660eb97e4 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Apr 2023 11:53:02 +1100 Subject: [PATCH 10/13] fix gdb path --- dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 989772fcaae..023a7332060 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -74,7 +74,7 @@ export READ_RECIPE_META_YAML_WHY_NOT=OFF # for available switches see # https://github.com/apache/arrow/blame/apache-arrow-11.0.0/cpp/cmake_modules/DefineOptions.cmake -# ARROW_GDB_INSTALL_DIR must match what's used for replacement in activate.sh +# placeholder in ARROW_GDB_INSTALL_DIR must match what's used for replacement in activate.sh cmake -GNinja \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ @@ -93,7 +93,7 @@ cmake -GNinja \ -DARROW_GANDIVA=ON \ -DARROW_GANDIVA_PC_CXX_FLAGS="${ARROW_GANDIVA_PC_CXX_FLAGS}" \ -DARROW_GCS=ON \ - -DARROW_GDB_INSTALL_DIR=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX \ + -DARROW_GDB_INSTALL_DIR=$PREFIX/share/gdb/auto-load/replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_JSON=ON \ From 4594fd72b14c63d34f63b76c74f5a0db6ab74da9 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 7 Apr 2023 21:04:13 +1100 Subject: [PATCH 11/13] fix gdb path again --- dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 023a7332060..fb8cbade865 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -93,7 +93,7 @@ cmake -GNinja \ -DARROW_GANDIVA=ON \ -DARROW_GANDIVA_PC_CXX_FLAGS="${ARROW_GANDIVA_PC_CXX_FLAGS}" \ -DARROW_GCS=ON \ - -DARROW_GDB_INSTALL_DIR=$PREFIX/share/gdb/auto-load/replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX \ + -DARROW_GDB_INSTALL_DIR=replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX/lib \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_JSON=ON \ From cbd13db3898b15a0f356f70cfea0d30d56eb32fb Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Sat, 8 Apr 2023 12:56:56 +1100 Subject: [PATCH 12/13] fix compiler dep for pyarrow-tests on linux; skip test on osx --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 18311229e54..fe917d0c292 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -328,10 +328,10 @@ outputs: test: requires: # test_cpp_extension_in_python requires a compiler - - {{ compiler("c") }} # [unix] + - {{ compiler("cxx") }} # [linux] - pytest - pytest-lazy-fixture - - backports.zoneinfo # [py<39] + - backports.zoneinfo # [py<39] - cffi - cloudpickle - cython @@ -364,6 +364,8 @@ outputs: # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] + # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our older macos sdk + {% set tests_to_skip = tests_to_skip + " or test_cpp_extension_in_python" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup {% set tests_to_skip = tests_to_skip + " or test_cython_api" %} # [unix] {% set tests_to_skip = tests_to_skip + " or test_visit_strings" %} # [unix] From 30da1a6af0c8b8f2f2eaedaafc0f67f040575c5c Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 10 Apr 2023 19:17:13 +1100 Subject: [PATCH 13/13] skip pyarrow tests when in emulation --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index fe917d0c292..2f79bbe958c 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -388,7 +388,7 @@ outputs: # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough - - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or (py==310)] + - pytest -v -rfEs -k "not ({{ tests_to_skip }})" # [linux64 or (py==310 and build_platform==target_platform)] about: home: http://github.com/apache/arrow