From a19b8a7093067e38dd3918226e8acdea28eebc6d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 8 Sep 2023 16:15:20 +1100 Subject: [PATCH 1/9] update arrow-cpp recipe --- dev/tasks/conda-recipes/arrow-cpp/activate.sh | 17 +++++-- .../conda-recipes/arrow-cpp/build-arrow.sh | 2 +- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 46 ++++++------------- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh b/dev/tasks/conda-recipes/arrow-cpp/activate.sh index 8757612781b..19d037ff412 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/activate.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh @@ -23,6 +23,13 @@ _la_log "Beginning libarrow activation." # where the GDB wrappers get installed _la_gdb_prefix="$CONDA_PREFIX/share/gdb/auto-load" +# If the directory is not writable, nothing can be done +if [ ! -w "$_la_gdb_prefix" ]; then + _la_log 'No rights to modify $_la_gdb_prefix, cannot create symlink!' + _la_log 'Unless you plan to use the GDB debugger with libarrow, this warning can be safely ignored.' + return +fi + # this needs to be in sync with ARROW_GDB_INSTALL_DIR in build.sh _la_placeholder="replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX" # the paths here are intentionally stacked, see #935, resp. @@ -44,7 +51,7 @@ for _la_target in "$_la_orig_install_dir/"*.py; do # If the file doesn't exist, skip this iteration of the loop. # (This happens when no files are found, in which case the # loop runs with target equal to the pattern itself.) - _la_log 'Folder $_la_orig_install_dir seems to not contain .py files, skipping' + _la_log 'Folder $_la_orig_install_dir seems to not contain .py files, skipping.' continue fi _la_symlink="$_la_symlink_dir/$(basename "$_la_target")" @@ -54,13 +61,13 @@ for _la_target in "$_la_orig_install_dir/"*.py; do _la_log 'symlink $_la_symlink already exists and points to $_la_target, skipping.' continue fi - _la_log 'Creating symlink $_la_symlink pointing to $_la_target' + _la_log 'Creating symlink $_la_symlink pointing to $_la_target.' mkdir -p "$_la_symlink_dir" || true # this check also creates the symlink; if it fails, we enter the if-branch. if ! ln -sf "$_la_target" "$_la_symlink"; then - echo -n "${BASH_SOURCE[0]} ERROR: Failed to create symlink from " - echo -n "'$_la_target' to '$_la_symlink'" - echo + echo -n "${BASH_SOURCE[0]} WARNING: Failed to create symlink from " + echo "'$_la_target' to '$_la_symlink'!" + echo "Unless you plan to use the GDB debugger with libarrow, this warning can be safely ignored." continue fi done diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index dc588f94738..629a9309421 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -43,8 +43,8 @@ if [[ "${build_platform}" != "${target_platform}" ]]; then fi EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang" EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link" + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_JEMALLOC_LG_PAGE=16" sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt - sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=16";g' ../cmake_modules/ThirdpartyToolchain.cmake fi # disable -fno-plt, which causes problems with GCC on PPC diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index ac4b29eb5ee..eb19857ae8f 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -61,7 +61,7 @@ outputs: build: string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} run_exports: - - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} + - {{ pin_subpackage("libarrow", max_pin="x") }} ignore_run_exports_from: - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] # arrow only uses headers, apparently @@ -133,8 +133,6 @@ outputs: # its host deps (which aren't yet covered above) leak into the build here - libcrc32c # [win] - libcurl # [win] - # same for libgrpc (before 1.55.0, which is coupled with libprotobuf 4.23.x) - - c-ares # [win and libprotobuf == "3.21"] run_constrained: - apache-arrow-proc =*={{ build_ext }} # make sure we don't co-install with old version of old package name @@ -198,8 +196,6 @@ outputs: requirements: host: - {{ pin_subpackage('libarrow', exact=True) }} - # avoid wrappers for different builds colliding due to identical hashes - - libprotobuf run: - {{ pin_subpackage('libarrow', exact=True) }} test: @@ -235,9 +231,7 @@ outputs: - cmake - ninja host: - # we're building for two protobuf versions, cannot pin exactly - # - {{ pin_subpackage('libarrow', exact=True) }} - - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} + - {{ pin_subpackage('libarrow', exact=True) }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - cython <3 @@ -246,8 +240,7 @@ outputs: - setuptools - setuptools_scm run: - # - {{ pin_subpackage('libarrow', exact=True) }} - - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} + - {{ pin_subpackage('libarrow', exact=True) }} - {{ pin_compatible('numpy') }} - python run_constrained: @@ -336,19 +329,15 @@ outputs: # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough {% if linux64 or py == 311 %} - # {% if not (aarch64 or ppc64le) or py in (310, 311) %} - # only run the full test suite for one python version when in emulation (each run takes ~45min); - # there's essentially zero divergence in behaviour across python versions anyway, and otherwise - # CUDA builds for aarch/ppc consistently run out of disk space on azure for some reason + # {% if not (aarch64 or ppc64le) or py == 311 %} + # only run the full test suite for one python version when in emulation + # (each run can take up to ~45min); there's essentially zero divergence + # in behaviour across python versions anyway test: requires: - # vary protobuf version in test suite (historically, test failures only have a very - # weak dependency on python version, so we don't lose coverage by doing half & half) - - libprotobuf <4 # [py % 2 == 0] # test_cpp_extension_in_python requires a compiler - {{ compiler("cxx") }} # [linux] - # temporary pin due to missing fixture - - pytest <7.4.0 + - pytest - pytest-lazy-fixture - backports.zoneinfo # [py<39] - cffi @@ -372,8 +361,6 @@ outputs: source_files: - testing/data commands: - - cd ${SP_DIR}/pyarrow/tests # [unix] - - cd %SP_DIR%\pyarrow\tests # [win] - export ARROW_TEST_DATA="${SRC_DIR}/testing/data" # [unix] - set "ARROW_TEST_DATA=%SRC_DIR%\testing\data" # [win] @@ -383,8 +370,6 @@ outputs: # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] - # tests that may crash the agent due to out-of-bound memory writes or other risky stuff - {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool" %} # [aarch64 or ppc64le] # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our older macos sdk {% set tests_to_skip = tests_to_skip + " or test_cpp_extension_in_python" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup @@ -394,22 +379,17 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv - {% set tests_to_skip = tests_to_skip + " or test_extension_to_pandas_storage_type" %} + # new fsspec changed behaviour, see https://github.com/apache/arrow/issues/37555 + {% set tests_to_skip = tests_to_skip + " or test_get_file_info_with_selector" %} # segfaults on OSX: to investigate ASAP {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] # gandiva tests are segfaulting on ppc {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] - # test failures on ppc + # test failures on ppc (both failing with: Float value was truncated converting to int32) {% set tests_to_skip = tests_to_skip + " or test_safe_cast_from_float_with_nans_to_int" %} # [ppc64le] - # gandiva tests are segfaulting on ppc - {% set tests_to_skip = tests_to_skip + " or test_float_with_null_as_integer" %} # [ppc64le] - # test is broken; header is in $PREFIX, not $SP_DIR - {% set tests_to_skip = tests_to_skip + " or (test_misc and test_get_include)" %} # [unix] - # flaky tests that fail occasionally - {% set tests_to_skip = tests_to_skip + " or test_total_bytes_allocated " %} # [linux] - {% set tests_to_skip = tests_to_skip + " or test_feather_format " %} # [linux] + {% set tests_to_skip = tests_to_skip + " or test_float_with_null_as_integer" %} # [ppc64le] # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ - - pytest -rfEs -k "not ({{ tests_to_skip }})" + - pytest --pyargs pyarrow -rfEs -k "not ({{ tests_to_skip }})" {% endif %} about: From af4c65db8400bc7f69f6560b48cc8c270f05d9c7 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 8 Sep 2023 16:16:24 +1100 Subject: [PATCH 2/9] update arrow-cpp ci_support files --- .../linux_64_cuda_compiler_version11.2.yaml | 14 ++++++-------- .../linux_64_cuda_compiler_versionNone.yaml | 14 ++++++-------- .../linux_aarch64_cuda_compiler_version11.2.yaml | 14 ++++++-------- .../linux_aarch64_cuda_compiler_versionNone.yaml | 14 ++++++-------- .../linux_ppc64le_cuda_compiler_version11.2.yaml | 14 ++++++-------- .../linux_ppc64le_cuda_compiler_versionNone.yaml | 14 ++++++-------- dev/tasks/conda-recipes/.ci_support/osx_64_.yaml | 14 ++++++-------- .../conda-recipes/.ci_support/osx_arm64_.yaml | 14 ++++++-------- .../win_64_cuda_compiler_version11.2.yaml | 16 ++++++---------- .../win_64_cuda_compiler_versionNone.yaml | 16 ++++++---------- 10 files changed, 60 insertions(+), 84 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml index 1cdcec199e7..0c9d3325e9a 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -35,18 +35,16 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index 5be5b58a739..d01a36daaae 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -35,18 +35,16 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml index 1677b03564c..57453f6d636 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml @@ -1,9 +1,9 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -39,18 +39,16 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -71,7 +69,7 @@ snappy: target_platform: - linux-aarch64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index 88fdf1254e6..d3024b99bfe 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -1,9 +1,9 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -39,18 +39,16 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -71,7 +69,7 @@ snappy: target_platform: - linux-aarch64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml index 3585db7b99b..8f9abe8cd4c 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -35,18 +35,16 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-ppc64le thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index c13a5222542..7655e309226 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -35,18 +35,16 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-ppc64le thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index dd4a230760e..8e6c0e4cb79 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,9 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -29,20 +29,18 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -63,7 +61,7 @@ snappy: target_platform: - osx-64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index 6a6713a54fe..f0ad4f06b26 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -1,9 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -29,20 +29,18 @@ google_cloud_cpp: libabseil: - '20230125' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 macos_machine: - arm64-apple-darwin20.0.0 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -63,7 +61,7 @@ snappy: target_platform: - osx-arm64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml index f75d92e276d..6b343887c2a 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml @@ -1,11 +1,9 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' -c_ares: -- '1' c_compiler: - vs2019 channel_sources: @@ -33,18 +31,16 @@ libcrc32c: libcurl: - '8' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -65,7 +61,7 @@ snappy: target_platform: - win-64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - cuda_compiler - cuda_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index 6d8fb15b15a..7fc386e8ed4 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,11 +1,9 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' -c_ares: -- '1' c_compiler: - vs2019 channel_sources: @@ -33,18 +31,16 @@ libcrc32c: libcurl: - '8' libgrpc: -- '1.54' - '1.56' libprotobuf: -- '3.21' - 4.23.3 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -65,7 +61,7 @@ snappy: target_platform: - win-64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - cuda_compiler - cuda_compiler_version From c04c1577fc6a9250465028629f9ebfbba5f9c463 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Fri, 8 Sep 2023 16:19:40 +1100 Subject: [PATCH 3/9] update r-arrow's ci_support files --- ...r_base4.1.yaml => linux_64_r_base4.3.yaml} | 2 +- ...e4.1.yaml => linux_aarch64_r_base4.3.yaml} | 2 +- ...4_r_base4.1.yaml => osx_64_r_base4.3.yaml} | 2 +- ..._base4.1.yaml => osx_arm64_r_base4.3.yaml} | 2 +- dev/tasks/tasks.yml | 24 +++++++++---------- 5 files changed, 16 insertions(+), 16 deletions(-) rename dev/tasks/conda-recipes/.ci_support/r/{linux_64_r_base4.1.yaml => linux_64_r_base4.3.yaml} (98%) rename dev/tasks/conda-recipes/.ci_support/r/{linux_aarch64_r_base4.1.yaml => linux_aarch64_r_base4.3.yaml} (98%) rename dev/tasks/conda-recipes/.ci_support/r/{osx_64_r_base4.1.yaml => osx_64_r_base4.3.yaml} (98%) rename dev/tasks/conda-recipes/.ci_support/r/{osx_arm64_r_base4.1.yaml => osx_arm64_r_base4.3.yaml} (98%) diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml index e63767cbe97..a4d06c9f20c 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - linux-64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml index 2b80b020fdc..028b190bb1e 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml @@ -23,7 +23,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - linux-aarch64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml index 6be6c2f5462..7b8b62d8e00 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - osx-64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml index 0ce856fcccf..a8e8aab83d5 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - osx-arm64 zip_keys: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index ed238778635..873d176b716 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -246,15 +246,15 @@ tasks: # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. - conda-linux-x64-cpu-r41: + conda-linux-x64-cpu-r43: ci: azure template: conda-recipes/azure.linux.yml params: config: linux_64_cuda_compiler_versionNone - r_config: linux_64_r_base4.1 + r_config: linux_64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-linux-x64-cpu-r42: ci: azure @@ -292,15 +292,15 @@ tasks: ########################### Conda Linux (aarch64) ########################### - conda-linux-aarch64-cpu-r41: + conda-linux-aarch64-cpu-r43: ci: azure template: conda-recipes/azure.linux.yml params: config: linux_aarch64_cuda_compiler_versionNone - r_config: linux_aarch64_r_base4.1 + r_config: linux_aarch64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-linux-aarch64-cpu-r42: ci: azure @@ -364,15 +364,15 @@ tasks: ############################## Conda OSX (x64) ############################## - conda-osx-x64-cpu-r41: + conda-osx-x64-cpu-r43: ci: azure template: conda-recipes/azure.osx.yml params: config: osx_64_ - r_config: osx_64_r_base4.1 + r_config: osx_64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-osx-x64-cpu-r42: ci: azure @@ -398,15 +398,15 @@ tasks: ############################# Conda OSX (arm64) ############################# - conda-osx-arm64-cpu-r41: + conda-osx-arm64-cpu-r43: ci: azure template: conda-recipes/azure.osx.yml params: config: osx_arm64_ - r_config: osx_arm64_r_base4.1 + r_config: osx_arm64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-osx-arm64-cpu-r42: ci: azure From f0b2133083f4afa7e9c476db88f609843e304296 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Wed, 13 Sep 2023 21:34:29 +1100 Subject: [PATCH 4/9] fixture teardown should not fail test --- python/pyarrow/tests/test_fs.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 8135f70f690..7aae2754a74 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -255,7 +255,10 @@ def s3fs(request, s3_server): allow_move_dir=False, allow_append_to_file=False, ) - fs.delete_dir(bucket) + try: + fs.delete_dir(bucket) + except OSError: + pass @pytest.fixture @@ -357,7 +360,10 @@ def py_fsspec_s3fs(request, s3_server): allow_move_dir=False, allow_append_to_file=True, ) - fs.delete_dir(bucket) + try: + fs.delete_dir(bucket) + except OSError: + pass @pytest.fixture(params=[ From 3584b5dbe0e4648e7d5631ab32adc006180bb5e1 Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 18 Sep 2023 09:20:48 +1100 Subject: [PATCH 5/9] sync recipe --- .../conda-recipes/arrow-cpp/build-arrow.sh | 2 +- .../conda-recipes/arrow-cpp/build-pyarrow.sh | 4 ++++ dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 24 ++++++++++++++----- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 629a9309421..ef0b038812a 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -30,7 +30,7 @@ fi # Enable CUDA support if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] then - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib" + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDAToolkit_ROOT=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib" else EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" fi diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 9c12321a1c1..f39e06874ca 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -24,6 +24,10 @@ BUILD_EXT_FLAGS="" # Enable CUDA support if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]; then export PYARROW_WITH_CUDA=1 + if [[ "${build_platform}" != "${target_platform}" ]]; then + export CUDAToolkit_ROOT=${CUDA_HOME} + export CMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib + fi else export PYARROW_WITH_CUDA=0 fi diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index eb19857ae8f..e90b7ccf8b5 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -114,6 +114,8 @@ outputs: - libgrpc - libprotobuf - libutf8proc + # gandiva requires shared libllvm + - llvm # [unix] - lz4-c - nlohmann_json # gandiva depends on openssl @@ -340,13 +342,16 @@ outputs: - pytest - pytest-lazy-fixture - backports.zoneinfo # [py<39] + - boto3 - cffi - cloudpickle - cython <3 - fastparquet - fsspec - hypothesis + - minio-server - pandas + - s3fs >=2023 - scipy # these are generally (far) behind on migrating abseil/grpc/protobuf, # and using them as test dependencies blocks the migrator unnecessarily @@ -361,6 +366,8 @@ outputs: source_files: - testing/data commands: + - cd ${SP_DIR} # [unix] + - cd %SP_DIR% # [win] - export ARROW_TEST_DATA="${SRC_DIR}/testing/data" # [unix] - set "ARROW_TEST_DATA=%SRC_DIR%\testing\data" # [win] @@ -370,26 +377,31 @@ outputs: # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] - # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our older macos sdk - {% set tests_to_skip = tests_to_skip + " or test_cpp_extension_in_python" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup {% set tests_to_skip = tests_to_skip + " or test_cython_api" %} # [unix] {% set tests_to_skip = tests_to_skip + " or test_visit_strings" %} # [unix] # skip tests that cannot succeed in emulation {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] + # XMinioInvalidObjectName on osx/win: "Object name contains unsupported characters" + {% set tests_to_skip = tests_to_skip + " or test_write_to_dataset_with_partitions_s3fs" %} # [osx or win] # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv + # currently broken + {% set tests_to_skip = tests_to_skip + " or test_fastparquet_cross_compatibility" %} # new fsspec changed behaviour, see https://github.com/apache/arrow/issues/37555 {% set tests_to_skip = tests_to_skip + " or test_get_file_info_with_selector" %} - # segfaults on OSX: to investigate ASAP - {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] + # problems with minio + {% set tests_to_skip = tests_to_skip + " or (test_delete_dir and S3FileSystem)" %} + {% set tests_to_skip = tests_to_skip + " or (test_delete_dir_contents and S3FileSystem)" %} + {% set tests_to_skip = tests_to_skip + " or (test_get_file_info and S3FileSystem)" %} + {% set tests_to_skip = tests_to_skip + " or (test_move_directory and S3FileSystem)" %} # gandiva tests are segfaulting on ppc - {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] + {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] # test failures on ppc (both failing with: Float value was truncated converting to int32) {% set tests_to_skip = tests_to_skip + " or test_safe_cast_from_float_with_nans_to_int" %} # [ppc64le] {% set tests_to_skip = tests_to_skip + " or test_float_with_null_as_integer" %} # [ppc64le] # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ - - pytest --pyargs pyarrow -rfEs -k "not ({{ tests_to_skip }})" + - pytest pyarrow/ -rfEs -k "not ({{ tests_to_skip }})" {% endif %} about: From d7af6149d822f21779bde31dae9097922f1832da Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Mon, 18 Sep 2023 09:22:12 +1100 Subject: [PATCH 6/9] update ci_support files (new abseil) --- .../.ci_support/linux_64_cuda_compiler_version11.2.yaml | 6 +++--- .../.ci_support/linux_64_cuda_compiler_versionNone.yaml | 6 +++--- .../linux_aarch64_cuda_compiler_version11.2.yaml | 6 +++--- .../linux_aarch64_cuda_compiler_versionNone.yaml | 6 +++--- .../linux_ppc64le_cuda_compiler_version11.2.yaml | 6 +++--- .../linux_ppc64le_cuda_compiler_versionNone.yaml | 6 +++--- dev/tasks/conda-recipes/.ci_support/osx_64_.yaml | 8 ++++---- dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml | 6 +++--- .../.ci_support/win_64_cuda_compiler_version11.2.yaml | 6 +++--- .../.ci_support/win_64_cuda_compiler_versionNone.yaml | 6 +++--- 10 files changed, 31 insertions(+), 31 deletions(-) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml index 0c9d3325e9a..042e2364d1c 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml @@ -33,11 +33,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index d01a36daaae..9885e6db38c 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -33,11 +33,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml index 57453f6d636..788b584504e 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml @@ -37,11 +37,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index d3024b99bfe..a1e4b8571ab 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -37,11 +37,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml index 8f9abe8cd4c..e21c4cbe853 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml @@ -33,11 +33,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index 7655e309226..89f1049ebdd 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -33,11 +33,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index 8e6c0e4cb79..2a5f8c5b36b 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,5 +1,5 @@ MACOSX_DEPLOYMENT_TARGET: -- '10.9' +- '10.13' aws_crt_cpp: - 0.23.1 aws_sdk_cpp: @@ -27,11 +27,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 macos_machine: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index f0ad4f06b26..211b71226ca 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -27,11 +27,11 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 macos_machine: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml index 6b343887c2a..32da33c0720 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml @@ -25,15 +25,15 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libcrc32c: - '1.1' libcurl: - '8' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index 7fc386e8ed4..6a33b86b9d6 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -25,15 +25,15 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libcrc32c: - '1.1' libcurl: - '8' libgrpc: -- '1.56' +- '1.57' libprotobuf: -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: From 53a4b70d8478cc6cf11e5df52751024b376f948d Mon Sep 17 00:00:00 2001 From: "H. Vetinari" Date: Thu, 21 Sep 2023 21:27:34 +1100 Subject: [PATCH 7/9] don't test with minio for now This reverts commit f0b2133083f4afa7e9c476db88f609843e304296. --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 8 ++------ python/pyarrow/tests/test_fs.py | 10 ++-------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index e90b7ccf8b5..6d10e140778 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -349,7 +349,8 @@ outputs: - fastparquet - fsspec - hypothesis - - minio-server + # currently disabled due to GH-37692 + # - minio-server - pandas - s3fs >=2023 - scipy @@ -390,11 +391,6 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_fastparquet_cross_compatibility" %} # new fsspec changed behaviour, see https://github.com/apache/arrow/issues/37555 {% set tests_to_skip = tests_to_skip + " or test_get_file_info_with_selector" %} - # problems with minio - {% set tests_to_skip = tests_to_skip + " or (test_delete_dir and S3FileSystem)" %} - {% set tests_to_skip = tests_to_skip + " or (test_delete_dir_contents and S3FileSystem)" %} - {% set tests_to_skip = tests_to_skip + " or (test_get_file_info and S3FileSystem)" %} - {% set tests_to_skip = tests_to_skip + " or (test_move_directory and S3FileSystem)" %} # gandiva tests are segfaulting on ppc {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] # test failures on ppc (both failing with: Float value was truncated converting to int32) diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py index 7aae2754a74..8135f70f690 100644 --- a/python/pyarrow/tests/test_fs.py +++ b/python/pyarrow/tests/test_fs.py @@ -255,10 +255,7 @@ def s3fs(request, s3_server): allow_move_dir=False, allow_append_to_file=False, ) - try: - fs.delete_dir(bucket) - except OSError: - pass + fs.delete_dir(bucket) @pytest.fixture @@ -360,10 +357,7 @@ def py_fsspec_s3fs(request, s3_server): allow_move_dir=False, allow_append_to_file=True, ) - try: - fs.delete_dir(bucket) - except OSError: - pass + fs.delete_dir(bucket) @pytest.fixture(params=[ From 7c0b94fc275c480881989a4870c0e60aa2e9619a Mon Sep 17 00:00:00 2001 From: h-vetinari Date: Mon, 25 Sep 2023 11:21:40 +0200 Subject: [PATCH 8/9] Remove test skip that's not necessary anymore. --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 6d10e140778..7a8cc989cb8 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -389,8 +389,6 @@ outputs: # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv # currently broken {% set tests_to_skip = tests_to_skip + " or test_fastparquet_cross_compatibility" %} - # new fsspec changed behaviour, see https://github.com/apache/arrow/issues/37555 - {% set tests_to_skip = tests_to_skip + " or test_get_file_info_with_selector" %} # gandiva tests are segfaulting on ppc {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] # test failures on ppc (both failing with: Float value was truncated converting to int32) From 68bf65c79ed9d8473b5874dfc482bf09a09438ef Mon Sep 17 00:00:00 2001 From: h-vetinari Date: Mon, 25 Sep 2023 11:26:32 +0200 Subject: [PATCH 9/9] Add test skip for risky test on osx --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 7a8cc989cb8..6d952b48b49 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -377,7 +377,8 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_cuda" %} # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] - {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] + # skip test that intentionally writes out of bounds and then expects no error message + {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled[system_memory_pool]" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup {% set tests_to_skip = tests_to_skip + " or test_cython_api" %} # [unix] {% set tests_to_skip = tests_to_skip + " or test_visit_strings" %} # [unix]