From 2e8f9962e7b1516febd695e7fde3b61f59293149 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 7 Apr 2021 12:37:24 +0200 Subject: [PATCH 01/11] ARROW-12246: [CI] Sync conda recipes with upstream feedstock --- ...on10.2numpy1.17python3.6.____cpython.yaml} | 35 +++++----- ...on10.2numpy1.17python3.7.____cpython.yaml} | 35 +++++----- ...on10.2numpy1.17python3.8.____cpython.yaml} | 35 +++++----- ...on10.2numpy1.19python3.9.____cpython.yaml} | 33 +++++----- ...onNonenumpy1.17python3.6.____cpython.yaml} | 33 +++++----- ...onNonenumpy1.17python3.7.____cpython.yaml} | 33 +++++----- ...onNonenumpy1.17python3.8.____cpython.yaml} | 33 +++++----- ...onNonenumpy1.19python3.9.____cpython.yaml} | 31 ++++----- dev/tasks/conda-recipes/build_steps.sh | 66 ++++++++++++------- dev/tasks/conda-recipes/run_docker_build.sh | 58 +++++++++++----- dev/tasks/tasks.yml | 16 ++--- 11 files changed, 231 insertions(+), 177 deletions(-) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.6.____cpython.yaml => linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml} (76%) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.7.____cpython.yaml => linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml} (76%) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.8.____cpython.yaml => linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml} (76%) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_version9.2python3.9.____cpython.yaml => linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml} (77%) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.6.____cpython.yaml => linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml} (77%) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.7.____cpython.yaml => linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml} (77%) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.8.____cpython.yaml => linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml} (77%) rename dev/tasks/conda-recipes/.ci_support/{linux_cuda_compiler_versionNonepython3.9.____cpython.yaml => linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml} (78%) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml similarity index 76% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml index 4c5061d6ff2..dd4c04197c9 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -15,30 +15,28 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '9.2' +- '10.2' cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-cuda:9.2 +- quay.io/condaforge/linux-anvil-cuda:10.2 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml similarity index 76% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml index db820f3ff26..f0c43929b56 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -15,30 +15,28 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '9.2' +- '10.2' cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-cuda:9.2 +- quay.io/condaforge/linux-anvil-cuda:10.2 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml similarity index 76% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml index 24810f3c0f1..149e70f438b 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -15,30 +15,28 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '9.2' +- '10.2' cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-cuda:9.2 +- quay.io/condaforge/linux-anvil-cuda:10.2 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml similarity index 77% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml index ebe422be367..fb15d4e7156 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_version9.2python3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -15,30 +15,28 @@ channel_targets: cuda_compiler: - nvcc cuda_compiler_version: -- '9.2' +- '10.2' cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-cuda:9.2 +- quay.io/condaforge/linux-anvil-cuda:10.2 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: - '1.19' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.9.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml similarity index 77% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml index 358814fb0da..d977f9e5779 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +19,24 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-comp7 +- quay.io/condaforge/linux-anvil-comp7 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml similarity index 77% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml index 054a730ee50..6ffa87a5eb9 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +19,24 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-comp7 +- quay.io/condaforge/linux-anvil-comp7 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml similarity index 77% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml index a3bfc58ea8f..7105f634953 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +19,24 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-comp7 +- quay.io/condaforge/linux-anvil-comp7 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml similarity index 78% rename from dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml index 88541395052..efe0148cc81 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_cuda_compiler_versionNonepython3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml @@ -1,13 +1,13 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - gcc c_compiler_version: -- '7' +- '9' +cdt_name: +- cos6 channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +19,24 @@ cuda_compiler_version: cxx_compiler: - gxx cxx_compiler_version: -- '7' +- '9' docker_image: -- condaforge/linux-anvil-comp7 +- quay.io/condaforge/linux-anvil-comp7 gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: - '1.19' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,18 +49,21 @@ pin_run_as_build: python: - 3.9.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - linux-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version - - cuda_compiler_version + - cdt_name - docker_image -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 8f1743f5946..7fbce051fd4 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -6,46 +6,68 @@ # benefit from the improvement. set -xeuo pipefail +export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +source ${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh + -output_dir=${1} +endgroup "Start Docker" +startgroup "Configuring conda" export PYTHONUNBUFFERED=1 -export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" cat >~/.condarc </dev/null && pwd )" -ARROW_ROOT=$(cd "$THISDIR/../../.."; pwd;) -FEEDSTOCK_ROOT=$THISDIR +PROVIDER_DIR="$(basename $THISDIR)" + +FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) +RECIPE_ROOT="${FEEDSTOCK_ROOT}/recipe" + +if [ -z ${FEEDSTOCK_NAME} ]; then + export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT}) +fi docker info @@ -25,6 +33,8 @@ if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) fi +ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts" + if [ -z "$CONFIG" ]; then set +x FILES=`ls .ci_support/linux_*` @@ -39,35 +49,49 @@ fi if [ -z "${DOCKER_IMAGE}" ]; then SHYAML_INSTALLED="$(shyaml -h || echo NO)" if [ "${SHYAML_INSTALLED}" == "NO" ]; then - echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to condaforge/linux-anvil-comp7" - DOCKER_IMAGE="condaforge/linux-anvil-comp7" + echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Trying to parse with coreutils" + DOCKER_IMAGE=$(cat .ci_support/${CONFIG}.yaml | grep '^docker_image:$' -A 1 | tail -n 1 | cut -b 3-) + if [ "${DOCKER_IMAGE}" = "" ]; then + echo "No docker_image entry found in ${CONFIG}. Falling back to condaforge/linux-anvil-comp7" + DOCKER_IMAGE="condaforge/linux-anvil-comp7" + fi else DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" fi fi -mkdir -p "${build_dir}" -DONE_CANARY="${build_dir}/conda-forge-build-done-${CONFIG}" +mkdir -p "$ARTIFACTS" +DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" rm -f "$DONE_CANARY" +# Allow people to specify extra default arguments to `docker run` (e.g. `--rm`) +DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ -z "${CI}" ]; then - DOCKER_RUN_ARGS="-it " + DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" fi +endgroup "Configure Docker" +startgroup "Start Docker" export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ - --shm-size=2G \ - -v "${ARROW_ROOT}":/arrow:rw,z \ - -v "${build_dir}":/build:rw \ - -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ + -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ + -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z,delegated \ -e CONFIG \ - -e R_CONFIG \ -e HOST_USER_ID \ -e UPLOAD_PACKAGES \ - -e ARROW_VERSION \ + -e GIT_BRANCH \ + -e UPLOAD_ON_BRANCH \ -e CI \ + -e FEEDSTOCK_NAME \ + -e CPU_COUNT \ + -e BUILD_WITH_CONDA_DEBUG \ + -e BUILD_OUTPUT_ID \ + -e BINSTAR_TOKEN \ + -e FEEDSTOCK_TOKEN \ + -e STAGING_BINSTAR_TOKEN \ $DOCKER_IMAGE \ - bash /arrow/dev/tasks/conda-recipes/build_steps.sh /build + bash \ + /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh # verify that the end of the script was reached -test -f "$DONE_CANARY" +test -f "$DONE_CANARY" \ No newline at end of file diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 5a04c98f640..9502b546dcd 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -166,7 +166,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_versionNonepython3.6.____cpython + config: linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython r_config: linux_64_r_base3.6 artifacts: - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -176,7 +176,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_versionNonepython3.7.____cpython + config: linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython r_config: linux_64_r_base4.0 artifacts: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -186,7 +186,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_versionNonepython3.8.____cpython + config: linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython artifacts: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -195,7 +195,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_versionNonepython3.9.____cpython + config: linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython artifacts: - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -204,7 +204,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_version9.2python3.6.____cpython + config: linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython artifacts: - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cuda.tar.bz2 @@ -213,7 +213,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_version9.2python3.7.____cpython + config: linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython artifacts: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cuda.tar.bz2 @@ -222,7 +222,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_version9.2python3.8.____cpython + config: linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython artifacts: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cuda.tar.bz2 @@ -231,7 +231,7 @@ tasks: ci: azure template: conda-recipes/azure.linux.yml params: - config: linux_cuda_compiler_version9.2python3.9.____cpython + config: linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython artifacts: - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cuda.tar.bz2 From f1fb2e5f89d063288aa47d6f2933d806d71720b1 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 7 Apr 2021 13:51:53 +0200 Subject: [PATCH 02/11] add logging_utils script --- .../conda-recipes/.scripts/logging_utils.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 dev/tasks/conda-recipes/.scripts/logging_utils.sh diff --git a/dev/tasks/conda-recipes/.scripts/logging_utils.sh b/dev/tasks/conda-recipes/.scripts/logging_utils.sh new file mode 100644 index 00000000000..a53ef3f2c7a --- /dev/null +++ b/dev/tasks/conda-recipes/.scripts/logging_utils.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Provide a unified interface for the different logging +# utilities CI providers offer. If unavailable, provide +# a compatible fallback (e.g. bare `echo xxxxxx`). + +function startgroup { + # Start a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[group]$1";; + travis ) + echo "$1" + echo -en 'travis_fold:start:'"${1// /}"'\\r';; + * ) + echo "$1";; + esac +} + +function endgroup { + # End a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[endgroup]";; + travis ) + echo -en 'travis_fold:end:'"${1// /}"'\\r';; + esac +} From f9b2a217f115d6ba234e14dda51ab8f3baae42f6 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 7 Apr 2021 17:14:38 +0200 Subject: [PATCH 03/11] update recipe as well --- ...solve-namespace-collisions-with-prot.patch | 92 ++++++ dev/tasks/conda-recipes/arrow-cpp/9569.patch | 283 ++++++++++++++++++ .../conda-recipes/arrow-cpp/bld-arrow.bat | 14 + .../conda-recipes/arrow-cpp/bld-pyarrow.bat | 30 +- .../conda-recipes/arrow-cpp/build-arrow.sh | 39 ++- .../conda-recipes/arrow-cpp/build-pyarrow.sh | 14 +- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 106 ++++--- 7 files changed, 496 insertions(+), 82 deletions(-) create mode 100644 dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch create mode 100644 dev/tasks/conda-recipes/arrow-cpp/9569.patch mode change 100755 => 100644 dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh mode change 100755 => 100644 dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh diff --git a/dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch b/dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch new file mode 100644 index 00000000000..7d4f3f7f5de --- /dev/null +++ b/dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch @@ -0,0 +1,92 @@ +From 5ef559bb17786cf4c57908ca086651a93577407f Mon Sep 17 00:00:00 2001 +From: "Uwe L. Korn" +Date: Sun, 21 Feb 2021 22:05:23 +0100 +Subject: [PATCH] ARROW-11724: [C++] Resolve namespace collisions with protobuf + 3.15 + +--- + cpp/src/arrow/flight/client.cc | 4 ++-- + cpp/src/arrow/flight/flight_test.cc | 4 ++-- + cpp/src/arrow/flight/serialization_internal.cc | 4 ++-- + cpp/src/arrow/flight/server.cc | 4 ++-- + 4 files changed, 8 insertions(+), 8 deletions(-) + +diff --git a/cpp/src/arrow/flight/client.cc b/cpp/src/arrow/flight/client.cc +index f42fbdaa9..724f999fa 100644 +--- a/cpp/src/arrow/flight/client.cc ++++ b/cpp/src/arrow/flight/client.cc +@@ -58,12 +58,12 @@ + #include "arrow/flight/serialization_internal.h" + #include "arrow/flight/types.h" + +-namespace pb = arrow::flight::protocol; +- + namespace arrow { + + namespace flight { + ++namespace pb = arrow::flight::protocol; ++ + const char* kWriteSizeDetailTypeId = "flight::FlightWriteSizeStatusDetail"; + + FlightCallOptions::FlightCallOptions() +diff --git a/cpp/src/arrow/flight/flight_test.cc b/cpp/src/arrow/flight/flight_test.cc +index 663fead62..2cea4c490 100644 +--- a/cpp/src/arrow/flight/flight_test.cc ++++ b/cpp/src/arrow/flight/flight_test.cc +@@ -51,11 +51,11 @@ + #include "arrow/flight/middleware_internal.h" + #include "arrow/flight/test_util.h" + +-namespace pb = arrow::flight::protocol; +- + namespace arrow { + namespace flight { + ++namespace pb = arrow::flight::protocol; ++ + const char kValidUsername[] = "flight_username"; + const char kValidPassword[] = "flight_password"; + const char kInvalidUsername[] = "invalid_flight_username"; +diff --git a/cpp/src/arrow/flight/serialization_internal.cc b/cpp/src/arrow/flight/serialization_internal.cc +index cf5f4140c..8c6b737c7 100644 +--- a/cpp/src/arrow/flight/serialization_internal.cc ++++ b/cpp/src/arrow/flight/serialization_internal.cc +@@ -53,14 +53,14 @@ + #include "arrow/util/bit_util.h" + #include "arrow/util/logging.h" + +-namespace pb = arrow::flight::protocol; +- + static constexpr int64_t kInt32Max = std::numeric_limits::max(); + + namespace arrow { + namespace flight { + namespace internal { + ++namespace pb = arrow::flight::protocol; ++ + using arrow::ipc::IpcPayload; + + using google::protobuf::internal::WireFormatLite; +diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc +index 4e35950c8..d1b4e0fa8 100644 +--- a/cpp/src/arrow/flight/server.cc ++++ b/cpp/src/arrow/flight/server.cc +@@ -60,11 +60,11 @@ using ServerContext = grpc::ServerContext; + template + using ServerWriter = grpc::ServerWriter; + +-namespace pb = arrow::flight::protocol; +- + namespace arrow { + namespace flight { + ++namespace pb = arrow::flight::protocol; ++ + // Macro that runs interceptors before returning the given status + #define RETURN_WITH_MIDDLEWARE(CONTEXT, STATUS) \ + do { \ +-- +2.24.3 (Apple Git-128) + diff --git a/dev/tasks/conda-recipes/arrow-cpp/9569.patch b/dev/tasks/conda-recipes/arrow-cpp/9569.patch new file mode 100644 index 00000000000..48eb752ce79 --- /dev/null +++ b/dev/tasks/conda-recipes/arrow-cpp/9569.patch @@ -0,0 +1,283 @@ +From e768c3c1a480e7aa35012d8f92f6e031eddba6ca Mon Sep 17 00:00:00 2001 +From: David Li +Date: Mon, 22 Feb 2021 17:20:29 -0500 +Subject: [PATCH] ARROW-11695: [C++][FlightRPC] fix option to disable TLS + verification + +--- + cpp/src/arrow/flight/CMakeLists.txt | 87 ++++++++++++------- + cpp/src/arrow/flight/client.cc | 38 ++++++-- + .../flight/try_compile/check_tls_opts_134.cc | 44 ++++++++++ + .../flight/try_compile/check_tls_opts_136.cc | 38 ++++++++ + 4 files changed, 166 insertions(+), 41 deletions(-) + create mode 100644 cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc + create mode 100644 cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc + +diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt +index 2fcb6ef077d..b44bab29074 100644 +--- a/cpp/src/arrow/flight/CMakeLists.txt ++++ b/cpp/src/arrow/flight/CMakeLists.txt +@@ -73,44 +73,67 @@ string(REPLACE "-Werror " " " CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + + # Probe the version of gRPC being used to see if it supports disabling server + # verification when using TLS. +-if(NOT DEFINED HAS_GRPC_132) +- message(STATUS "Checking support for TlsCredentialsOptions...") +- get_property(CURRENT_INCLUDE_DIRECTORIES +- DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +- PROPERTY INCLUDE_DIRECTORIES) +- try_compile(HAS_GRPC_132 ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES +- "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/check_tls_opts_132.cc" +- CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CURRENT_INCLUDE_DIRECTORIES}" +- LINK_LIBRARIES gRPC::grpc +- OUTPUT_VARIABLE TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11) +- +- if(HAS_GRPC_132) +- message(STATUS "TlsCredentialsOptions found in grpc::experimental.") +- add_definitions(-DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) +- else() +- message(STATUS "TlsCredentialsOptions not found in grpc::experimental.") +- message(DEBUG "Build output:") +- list(APPEND CMAKE_MESSAGE_INDENT "check_tls_opts_132.cc: ") +- message(DEBUG ${TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT}) +- list(REMOVE_AT CMAKE_MESSAGE_INDENT -1) +- +- try_compile(HAS_GRPC_127 ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES +- "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/check_tls_opts_127.cc" ++function(test_grpc_version DST_VAR DETECT_VERSION TEST_FILE) ++ if(NOT DEFINED ${DST_VAR}) ++ message( ++ STATUS "Checking support for TlsCredentialsOptions (gRPC >= ${DETECT_VERSION})...") ++ get_property(CURRENT_INCLUDE_DIRECTORIES ++ DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ++ PROPERTY INCLUDE_DIRECTORIES) ++ try_compile(HAS_GRPC_VERSION ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES ++ "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/${TEST_FILE}" + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CURRENT_INCLUDE_DIRECTORIES}" +- OUTPUT_VARIABLE TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11) +- +- if(HAS_GRPC_127) +- message(STATUS "TlsCredentialsOptions found in grpc_impl::experimental.") +- add_definitions( +- -DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc_impl::experimental) ++ LINK_LIBRARIES gRPC::grpc gRPC::grpc++ ++ OUTPUT_VARIABLE TLS_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11) ++ message(STATUS "${TLS_CREDENTIALS_OPTIONS_CHECK_OUTPUT}") ++ if(HAS_GRPC_VERSION) ++ set(${DST_VAR} "${DETECT_VERSION}" PARENT_SCOPE) + else() +- message(STATUS "TlsCredentialsOptions not found in grpc_impl::experimental.") ++ message( ++ STATUS ++ "TlsCredentialsOptions (for gRPC ${DETECT_VERSION}) not found in grpc::experimental." ++ ) + message(DEBUG "Build output:") +- list(APPEND CMAKE_MESSAGE_INDENT "check_tls_opts_127.cc: ") +- message(DEBUG ${TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT}) ++ list(APPEND CMAKE_MESSAGE_INDENT "${TEST_FILE}: ") ++ message(DEBUG ${TLS_CREDENTIALS_OPTIONS_CHECK_OUTPUT}) + list(REMOVE_AT CMAKE_MESSAGE_INDENT -1) + endif() + endif() ++endfunction() ++ ++test_grpc_version(GRPC_VERSION "1.36" "check_tls_opts_136.cc") ++test_grpc_version(GRPC_VERSION "1.34" "check_tls_opts_134.cc") ++test_grpc_version(GRPC_VERSION "1.32" "check_tls_opts_132.cc") ++test_grpc_version(GRPC_VERSION "1.27" "check_tls_opts_127.cc") ++message( ++ STATUS ++ "Found approximate gRPC version: ${GRPC_VERSION} (ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=${ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS})" ++ ) ++if(GRPC_VERSION EQUAL "1.27") ++ add_definitions(-DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc_impl::experimental) ++elseif(GRPC_VERSION EQUAL "1.32") ++ add_definitions(-DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) ++elseif(GRPC_VERSION EQUAL "1.34") ++ add_definitions(-DGRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS ++ -DGRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS_ROOT_CERTS ++ -DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) ++elseif(GRPC_VERSION EQUAL "1.36") ++ add_definitions(-DGRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS ++ -DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) ++else() ++ message( ++ STATUS ++ "A proper version of gRPC could not be found to support TlsCredentialsOptions in Arrow Flight." ++ ) ++ message( ++ STATUS ++ "You may need a newer version of gRPC (>= 1.27), or the gRPC API has changed and Flight must be updated to match." ++ ) ++ if(ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS) ++ message( ++ FATAL_ERROR "Halting build since ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS is set." ++ ) ++ endif() + endif() + + # Restore the CXXFLAGS that were modified above +diff --git a/cpp/src/arrow/flight/client.cc b/cpp/src/arrow/flight/client.cc +index 724f999fabe..cd6f3a97e58 100644 +--- a/cpp/src/arrow/flight/client.cc ++++ b/cpp/src/arrow/flight/client.cc +@@ -860,7 +860,7 @@ namespace { + // requires root CA certs, even if you are skipping server + // verification. + #if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) +-constexpr char BLANK_ROOT_PEM[] = ++constexpr char kDummyRootCert[] = + "-----BEGIN CERTIFICATE-----\n" + "MIICwzCCAaugAwIBAgIJAM12DOkcaqrhMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV\n" + "BAMTCWxvY2FsaG9zdDAeFw0yMDEwMDcwODIyNDFaFw0zMDEwMDUwODIyNDFaMBQx\n" +@@ -893,11 +893,7 @@ class FlightClient::FlightClientImpl { + + if (scheme == kSchemeGrpcTls) { + if (options.disable_server_verification) { +-#if !defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) +- return Status::NotImplemented( +- "Using encryption with server verification disabled is unsupported. " +- "Please use a release of Arrow Flight built with gRPC 1.27 or higher."); +-#else ++#if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) + namespace ge = GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS; + + // A callback to supply to TlsCredentialsOptions that accepts any server +@@ -910,16 +906,40 @@ class FlightClient::FlightClientImpl { + return 0; + } + }; +- ++ auto server_authorization_check = std::make_shared(); + noop_auth_check_ = std::make_shared( +- std::make_shared()); ++ server_authorization_check); ++#if defined(GRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS) ++ auto certificate_provider = ++ std::make_shared( ++ kDummyRootCert); ++#if defined(GRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS_ROOT_CERTS) ++ grpc::experimental::TlsChannelCredentialsOptions tls_options( ++ certificate_provider); ++#else ++ // While gRPC >= 1.36 does not require a root cert (it has a default) ++ // in practice the path it hardcodes is broken. See grpc/grpc#21655. ++ grpc::experimental::TlsChannelCredentialsOptions tls_options; ++ tls_options.set_certificate_provider(certificate_provider); ++#endif ++ tls_options.watch_root_certs(); ++ tls_options.set_root_cert_name("dummy"); ++ tls_options.set_server_verification_option( ++ grpc_tls_server_verification_option::GRPC_TLS_SKIP_ALL_SERVER_VERIFICATION); ++ tls_options.set_server_authorization_check_config(noop_auth_check_); ++#elif defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) + auto materials_config = std::make_shared(); +- materials_config->set_pem_root_certs(BLANK_ROOT_PEM); ++ materials_config->set_pem_root_certs(kDummyRootCert); + ge::TlsCredentialsOptions tls_options( + GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE, + GRPC_TLS_SKIP_ALL_SERVER_VERIFICATION, materials_config, + std::shared_ptr(), noop_auth_check_); ++#endif + creds = ge::TlsCredentials(tls_options); ++#else ++ return Status::NotImplemented( ++ "Using encryption with server verification disabled is unsupported. " ++ "Please use a release of Arrow Flight built with gRPC 1.27 or higher."); + #endif + } else { + grpc::SslCredentialsOptions ssl_options; +diff --git a/cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc b/cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc +new file mode 100644 +index 00000000000..4ee2122ef57 +--- /dev/null ++++ b/cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc +@@ -0,0 +1,44 @@ ++// Licensed to the Apache Software Foundation (ASF) under one ++// or more contributor license agreements. See the NOTICE file ++// distributed with this work for additional information ++// regarding copyright ownership. The ASF licenses this file ++// to you under the Apache License, Version 2.0 (the ++// "License"); you may not use this file except in compliance ++// with the License. You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, ++// software distributed under the License is distributed on an ++// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY ++// KIND, either express or implied. See the License for the ++// specific language governing permissions and limitations ++// under the License. ++ ++// Dummy file for checking if TlsCredentialsOptions exists in ++// the grpc::experimental namespace. gRPC starting from 1.34 ++// put it here. This is for supporting disabling server ++// validation when using TLS. ++ ++#include ++#include ++#include ++ ++// Dummy file for checking if TlsCredentialsOptions exists in ++// the grpc::experimental namespace. gRPC starting from 1.34 ++// puts it here. This is for supporting disabling server ++// validation when using TLS. ++ ++static void check() { ++ // In 1.34, there's no parameterless constructor; in 1.36, there's ++ // only a parameterless constructor ++ auto options = ++ std::make_shared(nullptr); ++ options->set_server_verification_option( ++ grpc_tls_server_verification_option::GRPC_TLS_SERVER_VERIFICATION); ++} ++ ++int main(int argc, const char** argv) { ++ check(); ++ return 0; ++} +diff --git a/cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc b/cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc +new file mode 100644 +index 00000000000..638eec67ba7 +--- /dev/null ++++ b/cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc +@@ -0,0 +1,38 @@ ++// Licensed to the Apache Software Foundation (ASF) under one ++// or more contributor license agreements. See the NOTICE file ++// distributed with this work for additional information ++// regarding copyright ownership. The ASF licenses this file ++// to you under the Apache License, Version 2.0 (the ++// "License"); you may not use this file except in compliance ++// with the License. You may obtain a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, ++// software distributed under the License is distributed on an ++// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY ++// KIND, either express or implied. See the License for the ++// specific language governing permissions and limitations ++// under the License. ++ ++// Dummy file for checking if TlsCredentialsOptions exists in ++// the grpc::experimental namespace. gRPC starting from 1.36 ++// puts it here. This is for supporting disabling server ++// validation when using TLS. ++ ++#include ++#include ++#include ++ ++static void check() { ++ // In 1.34, there's no parameterless constructor; in 1.36, there's ++ // only a parameterless constructor ++ auto options = std::make_shared(); ++ options->set_server_verification_option( ++ grpc_tls_server_verification_option::GRPC_TLS_SERVER_VERIFICATION); ++} ++ ++int main(int argc, const char** argv) { ++ check(); ++ return 0; ++} diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat index cc2ed71fe3e..2cc6ed1ba3e 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat @@ -1,6 +1,19 @@ +@echo on + mkdir "%SRC_DIR%"\cpp\build pushd "%SRC_DIR%"\cpp\build +:: Enable CUDA support +if "%cuda_compiler_version%"=="None" ( + set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=OFF" +) else ( + REM this should move to nvcc-feedstock + set "CUDA_PATH=%CUDA_PATH:\=/%" + set "CUDA_HOME=%CUDA_HOME:\=/%" + + set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON" +) + cmake -G "Ninja" ^ -DBUILD_SHARED_LIBS=ON ^ -DCMAKE_INSTALL_PREFIX="%LIBRARY_PREFIX%" ^ @@ -31,6 +44,7 @@ cmake -G "Ninja" ^ -DARROW_S3:BOOL=ON ^ -DBoost_NO_BOOST_CMAKE=ON ^ -DCMAKE_UNITY_BUILD=ON ^ + %EXTRA_CMAKE_ARGS% ^ .. if errorlevel 1 exit 1 diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat index 8f3357748df..23a09149dad 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -1,19 +1,16 @@ -@echo on pushd "%SRC_DIR%"\python @rem the symlinks for cmake modules don't work here -@rem del cmake_modules\BuildUtils.cmake -@rem del cmake_modules\SetupCxxFlags.cmake -@rem del cmake_modules\FindNumPy.cmake -@rem del cmake_modules\FindPythonLibsNew.cmake -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ -@rem if errorlevel 1 exit 1 -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ -@rem if errorlevel 1 exit 1 -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ -@rem if errorlevel 1 exit 1 -@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ -@rem if errorlevel 1 exit 1 +del cmake_modules\BuildUtils.cmake +del cmake_modules\SetupCxxFlags.cmake +del cmake_modules\CompilerInfo.cmake +del cmake_modules\FindNumPy.cmake +del cmake_modules\FindPythonLibsNew.cmake +copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ +copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ SET ARROW_HOME=%LIBRARY_PREFIX% SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% @@ -26,6 +23,13 @@ SET PYARROW_WITH_GANDIVA=1 SET PYARROW_WITH_PARQUET=1 SET PYARROW_CMAKE_GENERATOR=Ninja +:: Enable CUDA support +if "%cuda_compiler_version%"=="None" ( + set "PYARROW_WITH_CUDA=0" +) else ( + set "PYARROW_WITH_CUDA=1" +) + %PYTHON% setup.py ^ build_ext ^ install --single-version-externally-managed ^ diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh old mode 100755 new mode 100644 index e15fc92808c..f9c1d975ec3 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -34,17 +34,26 @@ else EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" fi +if [[ "${target_platform}" == "osx-arm64" ]]; then + # We need llvm 11+ support in Arrow for this + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF" + sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt + sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=14";g' ../cmake_modules/ThirdpartyToolchain.cmake +else + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=ON" +fi + cmake \ - -DBUILD_SHARED_LIBS=ON \ -DARROW_BOOST_USE_SHARED=ON \ -DARROW_BUILD_BENCHMARKS=OFF \ -DARROW_BUILD_STATIC=OFF \ -DARROW_BUILD_TESTS=OFF \ -DARROW_BUILD_UTILITIES=OFF \ + -DBUILD_SHARED_LIBS=ON \ -DARROW_DATASET=ON \ -DARROW_DEPENDENCY_SOURCE=SYSTEM \ -DARROW_FLIGHT=ON \ - -DARROW_GANDIVA=ON \ + -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=ON \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_MIMALLOC=ON \ @@ -55,34 +64,32 @@ cmake \ -DARROW_PYTHON=ON \ -DARROW_S3=ON \ -DARROW_SIMD_LEVEL=NONE \ + -DARROW_USE_LD_GOLD=ON \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_LZ4=ON \ -DARROW_WITH_SNAPPY=ON \ -DARROW_WITH_ZLIB=ON \ -DARROW_WITH_ZSTD=ON \ - -DARROW_USE_LD_GOLD=ON \ - -DCMAKE_AR=${AR} \ -DCMAKE_BUILD_TYPE=release \ -DCMAKE_INSTALL_LIBDIR=lib \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ - -DCMAKE_RANLIB=${RANLIB} \ -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ - -DCMAKE_UNITY_BUILD=ON \ + -DPython3_EXECUTABLE=${PYTHON} \ + -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc \ -GNinja \ ${EXTRA_CMAKE_ARGS} \ .. -# Decrease parallelism a bit as we will otherwise get out-of-memory problems -# This is only necessary on Travis -if [ "${TRAVIS}" = "true" ]; then -# if [ "$(uname -m)" = "ppc64le" ]; then - echo "Using $(grep -c ^processor /proc/cpuinfo) CPUs" - CPU_COUNT=$(grep -c ^processor /proc/cpuinfo) - CPU_COUNT=$((CPU_COUNT / 4)) - ninja install -j${CPU_COUNT} -else - ninja install +# Commented out until jemalloc and mimalloc are fixed upstream +if [[ "${target_platform}" == "osx-arm64" ]]; then + ninja jemalloc_ep-prefix/src/jemalloc_ep-stamp/jemalloc_ep-patch mimalloc_ep-prefix/src/mimalloc_ep-stamp/mimalloc_ep-patch + cp $BUILD_PREFIX/share/gnuconfig/config.* jemalloc_ep-prefix/src/jemalloc_ep/build-aux/ + sed -ie 's/list(APPEND mi_cflags -march=native)//g' mimalloc_ep-prefix/src/mimalloc_ep/CMakeLists.txt + # Use the correct register for thread-local storage + sed -ie 's/tpidr_el0/tpidrro_el0/g' mimalloc_ep-prefix/src/mimalloc_ep/include/mimalloc-internal.h fi +ninja install + popd diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh old mode 100755 new mode 100644 index d0fb55de580..a394e999f7b --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -9,10 +9,14 @@ export PARQUET_HOME=$PREFIX export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION export PYARROW_BUILD_TYPE=release export PYARROW_BUNDLE_ARROW_CPP_HEADERS=0 -export PYARROW_BUNDLE_PLASMA_EXECUTABLE=0 export PYARROW_WITH_DATASET=1 export PYARROW_WITH_FLIGHT=1 -export PYARROW_WITH_GANDIVA=1 +if [[ "${target_platform}" == "osx-arm64" ]]; then + # We need llvm 11+ support in Arrow for this + export PYARROW_WITH_GANDIVA=0 +else + export PYARROW_WITH_GANDIVA=1 +fi export PYARROW_WITH_HDFS=1 export PYARROW_WITH_ORC=1 export PYARROW_WITH_PARQUET=1 @@ -22,16 +26,14 @@ export PYARROW_CMAKE_GENERATOR=Ninja BUILD_EXT_FLAGS="" # Enable CUDA support -if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] -then +if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]; then export PYARROW_WITH_CUDA=1 else export PYARROW_WITH_CUDA=0 fi # Resolve: Make Error at cmake_modules/SetupCxxFlags.cmake:338 (message): Unsupported arch flag: -march=. -if [[ "$(uname -m)" = "aarch64" ]] -then +if [[ "${target_platform}" == "linux-aarch64" ]]; then export PYARROW_CMAKE_OPTIONS="-DARROW_ARMV8_ARCH=armv8-a" fi diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index cdbfc5d5b4d..b45d7bdbc9e 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -1,20 +1,26 @@ -{% set version = ARROW_VERSION %} -{% set number = "0" %} -{% set cuda_enabled = cuda_compiler_version is not undefined and cuda_compiler_version == '9.2' %} -{% set build_ext_version = "1.0.1" %} +{% set version = "3.0.0" %} +{% set cuda_enabled = cuda_compiler_version != "None" %} +{% set build_ext_version = "3.0.0" %} {% set build_ext = "cuda" if cuda_enabled else "cpu" %} -{% set proc_build_number = "1" %} +{% set proc_build_number = "0" %} package: name: arrow-cpp-ext version: {{ version }} source: - path: ../../../../ + url: https://dist.apache.org/repos/dist/release/arrow/arrow-{{ version }}/apache-arrow-{{ version }}.tar.gz + sha256: 73c2cc3be537aa1f3fd9490cfec185714168c9bfd599d23e287ab0cc0558e27a + patches: + - 0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch + - 9569.patch build: - number: 0 - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] + number: 10 + # for cuda on win/linux, building with 9.2 is enough to be compatible with all later versions, + # since arrow is only using libcuda, and not libcudart. + skip: true # [(win or linux) and cuda_compiler_version not in ("None", "10.2")] + skip: true # [osx and cuda_compiler_version != "None"] run_exports: - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} @@ -40,27 +46,31 @@ outputs: version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] run_exports: - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} ignore_run_exports: - cudatoolkit - # Gandiva only needs headers - - boost-cpp track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} requirements: build: - - cmake 3.16.* + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - cython # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - gnuconfig # [osx and arm64] + - libprotobuf + - grpc-cpp + - cmake - autoconf # [unix] - ninja - make # [unix] - {{ compiler('c') }} - {{ compiler('cxx') }} - - {{ compiler("cuda") }} # [cuda_compiler_version not in (undefined, "None")] + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - aws-sdk-cpp - - boost-cpp + - boost-cpp >=1.70 - brotli - bzip2 - c-ares @@ -68,8 +78,8 @@ outputs: - glog - grpc-cpp - libprotobuf - - clangdev 11 - - llvmdev 11 + - clangdev 10 # [not (osx and arm64)] + - llvmdev 10 # [not (osx and arm64)] - libutf8proc - lz4-c - numpy @@ -78,25 +88,15 @@ outputs: - rapidjson - re2 - snappy - - thrift-cpp >=0.11 + - thrift-cpp - zlib - zstd run: - {{ pin_compatible('numpy', lower_bound='1.16') }} - - aws-sdk-cpp - - brotli - - c-ares - - gflags - - glog - - grpc-cpp - - lz4-c - python - - re2 - - zlib - - zstd run_constrained: - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] + - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] about: home: http://github.com/apache/arrow @@ -111,7 +111,7 @@ outputs: - test -f $PREFIX/include/arrow/api.h # [unix] - test -f $PREFIX/include/arrow/flight/types.h # [unix] - test -f $PREFIX/include/plasma/client.h # [unix] - - test -f $PREFIX/include/gandiva/engine.h # [unix and not (aarch64 or ppc64le)] + - test -f $PREFIX/include/gandiva/engine.h # [unix and not (osx and arm64)] - test -f $PREFIX/include/parquet/api/reader.h # [unix] - if not exist %LIBRARY_INC%\\arrow\\api.h exit 1 # [win] - if not exist %LIBRARY_INC%\\gandiva\\engine.h exit 1 # [win] @@ -123,15 +123,16 @@ outputs: - test -f $PREFIX/lib/libarrow_flight.so # [linux] - test -f $PREFIX/lib/libarrow_python.so # [linux] - test -f $PREFIX/lib/libparquet.so # [linux] - - test -f $PREFIX/lib/libgandiva.so # [linux and not (aarch64 or ppc64le)] + - test -f $PREFIX/lib/libgandiva.so # [linux] - test -f $PREFIX/lib/libplasma.so # [linux] - {{ "- test %s -f $PREFIX/lib/libarrow_cuda.so" % (['!', ''][cuda_enabled]) }} # [linux] - {{ "- test %s -f $PREFIX/lib/libarrow_cuda.dylib" % (['!', ''][cuda_enabled]) }} # [osx] - {{ "- if %s exist %%PREFIX%%\\Library\\bin\\arrow_cuda.dll exit 1" % (['', 'not'][cuda_enabled]) }} # [win] + - test -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version != "None") and unix] + - test ! -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version == "None") and unix] + - if not exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version != "None") and win] + - if exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version == "None") and win] - test -f $PREFIX/lib/libarrow.dylib # [osx] - test -f $PREFIX/lib/libarrow_dataset.dylib # [osx] - test -f $PREFIX/lib/libarrow_python.dylib # [osx] - - test -f $PREFIX/lib/libgandiva.dylib # [osx] + - test -f $PREFIX/lib/libgandiva.dylib # [osx and not arm64] - test -f $PREFIX/lib/libparquet.dylib # [osx] - test -f $PREFIX/lib/libplasma.dylib # [osx] - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win] @@ -162,20 +163,23 @@ outputs: version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] ignore_run_exports: - cudatoolkit track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} requirements: build: - - cmake 3.16.* + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - cython # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - cmake - ninja - make # [unix] - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version not in (undefined, "None")] + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - {{ pin_subpackage('arrow-cpp', exact=True) }} - cython @@ -192,7 +196,7 @@ outputs: - python run_constrained: - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] + - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] about: home: http://github.com/apache/arrow @@ -206,7 +210,7 @@ outputs: - pyarrow - pyarrow.dataset - pyarrow.flight - - pyarrow.gandiva + - pyarrow.gandiva # [not (osx and arm64)] - pyarrow.orc # [unix] - pyarrow.parquet - pyarrow.plasma # [unix] @@ -215,31 +219,39 @@ outputs: - pyarrow._hdfs # We can only test importing cuda package but cannot run when a # CUDA device is not available, for instance, when building from CI. - - pyarrow.cuda # [cuda_compiler_version not in (undefined, "None")] + # On Windows, we cannot even do that due to `nvcuda.dll` not being found, see + # https://conda-forge.org/docs/maintainer/knowledge_base.html#nvcuda-dll-cannot-be-found-on-windows + # However, we check below for (at least) the presence of a correctly-compiled module + - pyarrow.cuda # [cuda_compiler_version != "None" and not win] commands: - - test ! -f ${SP_DIR}/pyarrow/plasma-store-server # [unix] - - test ! -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] - - if exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] + - test ! -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] + - if exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] + # Need to remove dot from PY_VER; %MYVAR:x=y% replaces "x" in %MYVAR% with "y" + - if not exist %SP_DIR%/pyarrow/_cuda.cp%PY_VER:.=%-win_amd64.pyd exit 1 # [win and cuda_compiler_version != "None"] + - name: pyarrow-tests script: build-pyarrow.sh # [not win] script: bld-pyarrow.bat # [win] version: {{ version }} build: string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} - skip: true # [cuda_compiler_version not in (undefined, "None", "9.2")] ignore_run_exports: - cudatoolkit track_features: {{ "- arrow-cuda" if cuda_enabled else "" }} requirements: build: - - cmake 3.16.* + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - cython # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - cmake - ninja - make # [unix] - {{ compiler('c') }} - {{ compiler('cxx') }} # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda - - {{ compiler("cuda") }} # [cuda_compiler_version not in (undefined, "None")] + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] host: - {{ pin_subpackage('arrow-cpp', exact=True) }} - {{ pin_subpackage('pyarrow', exact=True) }} @@ -254,7 +266,7 @@ outputs: - python run_constrained: - arrow-cpp-proc * {{ build_ext }} - - cudatoolkit >=9.2 # [cuda_compiler_version not in (undefined, "None")] + - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] about: home: http://github.com/apache/arrow From b4973fbe2d354ab9f146b3bd3dc95a44a6b72d6e Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 8 Apr 2021 12:29:57 +0200 Subject: [PATCH 04/11] Revert changes to build scripts --- dev/tasks/conda-recipes/build_steps.sh | 70 ++++++++------------- dev/tasks/conda-recipes/run_docker_build.sh | 62 +++++++----------- 2 files changed, 47 insertions(+), 85 deletions(-) diff --git a/dev/tasks/conda-recipes/build_steps.sh b/dev/tasks/conda-recipes/build_steps.sh index 7fbce051fd4..25864c08a70 100755 --- a/dev/tasks/conda-recipes/build_steps.sh +++ b/dev/tasks/conda-recipes/build_steps.sh @@ -1,73 +1,55 @@ #!/usr/bin/env bash +# NOTE: This script has been slightly adopted to suite the Apache Arrow / crossbow CI +# setup. The next time this is updated to the current version on conda-forge, +# you will also make this additions afterwards. + # PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here # will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent # changes to this script, consider a proposal to conda-smithy so that other feedstocks can also # benefit from the improvement. set -xeuo pipefail -export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" -source ${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh - -endgroup "Start Docker" +output_dir=${1} -startgroup "Configuring conda" export PYTHONUNBUFFERED=1 -export RECIPE_ROOT="${RECIPE_ROOT:-/home/conda/recipe_root}" +export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" cat >~/.condarc </dev/null && pwd )" -PROVIDER_DIR="$(basename $THISDIR)" - -FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) -RECIPE_ROOT="${FEEDSTOCK_ROOT}/recipe" +build_dir=${1} -if [ -z ${FEEDSTOCK_NAME} ]; then - export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT}) -fi +THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" +ARROW_ROOT=$(cd "$THISDIR/../../.."; pwd;) +FEEDSTOCK_ROOT=$THISDIR docker info @@ -33,8 +29,6 @@ if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) fi -ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts" - if [ -z "$CONFIG" ]; then set +x FILES=`ls .ci_support/linux_*` @@ -49,49 +43,35 @@ fi if [ -z "${DOCKER_IMAGE}" ]; then SHYAML_INSTALLED="$(shyaml -h || echo NO)" if [ "${SHYAML_INSTALLED}" == "NO" ]; then - echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Trying to parse with coreutils" - DOCKER_IMAGE=$(cat .ci_support/${CONFIG}.yaml | grep '^docker_image:$' -A 1 | tail -n 1 | cut -b 3-) - if [ "${DOCKER_IMAGE}" = "" ]; then - echo "No docker_image entry found in ${CONFIG}. Falling back to condaforge/linux-anvil-comp7" - DOCKER_IMAGE="condaforge/linux-anvil-comp7" - fi + echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to condaforge/linux-anvil-comp7" + DOCKER_IMAGE="condaforge/linux-anvil-comp7" else DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" fi fi -mkdir -p "$ARTIFACTS" -DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}" +mkdir -p "${build_dir}" +DONE_CANARY="${build_dir}/conda-forge-build-done-${CONFIG}" rm -f "$DONE_CANARY" -# Allow people to specify extra default arguments to `docker run` (e.g. `--rm`) -DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}" if [ -z "${CI}" ]; then - DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}" + DOCKER_RUN_ARGS="-it " fi -endgroup "Configure Docker" -startgroup "Start Docker" export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" docker run ${DOCKER_RUN_ARGS} \ - -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \ - -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z,delegated \ + --shm-size=2G \ + -v "${ARROW_ROOT}":/arrow:rw,z \ + -v "${build_dir}":/build:rw \ + -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ -e CONFIG \ + -e R_CONFIG \ -e HOST_USER_ID \ -e UPLOAD_PACKAGES \ - -e GIT_BRANCH \ - -e UPLOAD_ON_BRANCH \ + -e ARROW_VERSION \ -e CI \ - -e FEEDSTOCK_NAME \ - -e CPU_COUNT \ - -e BUILD_WITH_CONDA_DEBUG \ - -e BUILD_OUTPUT_ID \ - -e BINSTAR_TOKEN \ - -e FEEDSTOCK_TOKEN \ - -e STAGING_BINSTAR_TOKEN \ $DOCKER_IMAGE \ - bash \ - /home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh + bash /arrow/dev/tasks/conda-recipes/build_steps.sh /build # verify that the end of the script was reached -test -f "$DONE_CANARY" \ No newline at end of file +test -f "$DONE_CANARY" From 24517fdf0f442500ec0b1fa8627e0e91d35ca78f Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 8 Apr 2021 15:24:50 +0200 Subject: [PATCH 05/11] Use ARROW_VERSION again --- dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index b45d7bdbc9e..11b3307523c 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -1,6 +1,6 @@ -{% set version = "3.0.0" %} +{% set version = ARROW_VERSION %} {% set cuda_enabled = cuda_compiler_version != "None" %} -{% set build_ext_version = "3.0.0" %} +{% set build_ext_version = ARROW_VERSION %} {% set build_ext = "cuda" if cuda_enabled else "cpu" %} {% set proc_build_number = "0" %} From cae64e5dc116fa6d0d7942f6c71b7598d29e04e0 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 8 Apr 2021 15:35:51 +0200 Subject: [PATCH 06/11] Use source: ...... --- ...solve-namespace-collisions-with-prot.patch | 92 ------ dev/tasks/conda-recipes/arrow-cpp/9569.patch | 283 ------------------ dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 8 +- 3 files changed, 2 insertions(+), 381 deletions(-) delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch delete mode 100644 dev/tasks/conda-recipes/arrow-cpp/9569.patch diff --git a/dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch b/dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch deleted file mode 100644 index 7d4f3f7f5de..00000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch +++ /dev/null @@ -1,92 +0,0 @@ -From 5ef559bb17786cf4c57908ca086651a93577407f Mon Sep 17 00:00:00 2001 -From: "Uwe L. Korn" -Date: Sun, 21 Feb 2021 22:05:23 +0100 -Subject: [PATCH] ARROW-11724: [C++] Resolve namespace collisions with protobuf - 3.15 - ---- - cpp/src/arrow/flight/client.cc | 4 ++-- - cpp/src/arrow/flight/flight_test.cc | 4 ++-- - cpp/src/arrow/flight/serialization_internal.cc | 4 ++-- - cpp/src/arrow/flight/server.cc | 4 ++-- - 4 files changed, 8 insertions(+), 8 deletions(-) - -diff --git a/cpp/src/arrow/flight/client.cc b/cpp/src/arrow/flight/client.cc -index f42fbdaa9..724f999fa 100644 ---- a/cpp/src/arrow/flight/client.cc -+++ b/cpp/src/arrow/flight/client.cc -@@ -58,12 +58,12 @@ - #include "arrow/flight/serialization_internal.h" - #include "arrow/flight/types.h" - --namespace pb = arrow::flight::protocol; -- - namespace arrow { - - namespace flight { - -+namespace pb = arrow::flight::protocol; -+ - const char* kWriteSizeDetailTypeId = "flight::FlightWriteSizeStatusDetail"; - - FlightCallOptions::FlightCallOptions() -diff --git a/cpp/src/arrow/flight/flight_test.cc b/cpp/src/arrow/flight/flight_test.cc -index 663fead62..2cea4c490 100644 ---- a/cpp/src/arrow/flight/flight_test.cc -+++ b/cpp/src/arrow/flight/flight_test.cc -@@ -51,11 +51,11 @@ - #include "arrow/flight/middleware_internal.h" - #include "arrow/flight/test_util.h" - --namespace pb = arrow::flight::protocol; -- - namespace arrow { - namespace flight { - -+namespace pb = arrow::flight::protocol; -+ - const char kValidUsername[] = "flight_username"; - const char kValidPassword[] = "flight_password"; - const char kInvalidUsername[] = "invalid_flight_username"; -diff --git a/cpp/src/arrow/flight/serialization_internal.cc b/cpp/src/arrow/flight/serialization_internal.cc -index cf5f4140c..8c6b737c7 100644 ---- a/cpp/src/arrow/flight/serialization_internal.cc -+++ b/cpp/src/arrow/flight/serialization_internal.cc -@@ -53,14 +53,14 @@ - #include "arrow/util/bit_util.h" - #include "arrow/util/logging.h" - --namespace pb = arrow::flight::protocol; -- - static constexpr int64_t kInt32Max = std::numeric_limits::max(); - - namespace arrow { - namespace flight { - namespace internal { - -+namespace pb = arrow::flight::protocol; -+ - using arrow::ipc::IpcPayload; - - using google::protobuf::internal::WireFormatLite; -diff --git a/cpp/src/arrow/flight/server.cc b/cpp/src/arrow/flight/server.cc -index 4e35950c8..d1b4e0fa8 100644 ---- a/cpp/src/arrow/flight/server.cc -+++ b/cpp/src/arrow/flight/server.cc -@@ -60,11 +60,11 @@ using ServerContext = grpc::ServerContext; - template - using ServerWriter = grpc::ServerWriter; - --namespace pb = arrow::flight::protocol; -- - namespace arrow { - namespace flight { - -+namespace pb = arrow::flight::protocol; -+ - // Macro that runs interceptors before returning the given status - #define RETURN_WITH_MIDDLEWARE(CONTEXT, STATUS) \ - do { \ --- -2.24.3 (Apple Git-128) - diff --git a/dev/tasks/conda-recipes/arrow-cpp/9569.patch b/dev/tasks/conda-recipes/arrow-cpp/9569.patch deleted file mode 100644 index 48eb752ce79..00000000000 --- a/dev/tasks/conda-recipes/arrow-cpp/9569.patch +++ /dev/null @@ -1,283 +0,0 @@ -From e768c3c1a480e7aa35012d8f92f6e031eddba6ca Mon Sep 17 00:00:00 2001 -From: David Li -Date: Mon, 22 Feb 2021 17:20:29 -0500 -Subject: [PATCH] ARROW-11695: [C++][FlightRPC] fix option to disable TLS - verification - ---- - cpp/src/arrow/flight/CMakeLists.txt | 87 ++++++++++++------- - cpp/src/arrow/flight/client.cc | 38 ++++++-- - .../flight/try_compile/check_tls_opts_134.cc | 44 ++++++++++ - .../flight/try_compile/check_tls_opts_136.cc | 38 ++++++++ - 4 files changed, 166 insertions(+), 41 deletions(-) - create mode 100644 cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc - create mode 100644 cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc - -diff --git a/cpp/src/arrow/flight/CMakeLists.txt b/cpp/src/arrow/flight/CMakeLists.txt -index 2fcb6ef077d..b44bab29074 100644 ---- a/cpp/src/arrow/flight/CMakeLists.txt -+++ b/cpp/src/arrow/flight/CMakeLists.txt -@@ -73,44 +73,67 @@ string(REPLACE "-Werror " " " CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - - # Probe the version of gRPC being used to see if it supports disabling server - # verification when using TLS. --if(NOT DEFINED HAS_GRPC_132) -- message(STATUS "Checking support for TlsCredentialsOptions...") -- get_property(CURRENT_INCLUDE_DIRECTORIES -- DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -- PROPERTY INCLUDE_DIRECTORIES) -- try_compile(HAS_GRPC_132 ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES -- "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/check_tls_opts_132.cc" -- CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CURRENT_INCLUDE_DIRECTORIES}" -- LINK_LIBRARIES gRPC::grpc -- OUTPUT_VARIABLE TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11) -- -- if(HAS_GRPC_132) -- message(STATUS "TlsCredentialsOptions found in grpc::experimental.") -- add_definitions(-DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) -- else() -- message(STATUS "TlsCredentialsOptions not found in grpc::experimental.") -- message(DEBUG "Build output:") -- list(APPEND CMAKE_MESSAGE_INDENT "check_tls_opts_132.cc: ") -- message(DEBUG ${TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT}) -- list(REMOVE_AT CMAKE_MESSAGE_INDENT -1) -- -- try_compile(HAS_GRPC_127 ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES -- "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/check_tls_opts_127.cc" -+function(test_grpc_version DST_VAR DETECT_VERSION TEST_FILE) -+ if(NOT DEFINED ${DST_VAR}) -+ message( -+ STATUS "Checking support for TlsCredentialsOptions (gRPC >= ${DETECT_VERSION})...") -+ get_property(CURRENT_INCLUDE_DIRECTORIES -+ DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -+ PROPERTY INCLUDE_DIRECTORIES) -+ try_compile(HAS_GRPC_VERSION ${CMAKE_CURRENT_BINARY_DIR}/try_compile SOURCES -+ "${CMAKE_CURRENT_SOURCE_DIR}/try_compile/${TEST_FILE}" - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${CURRENT_INCLUDE_DIRECTORIES}" -- OUTPUT_VARIABLE TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11) -- -- if(HAS_GRPC_127) -- message(STATUS "TlsCredentialsOptions found in grpc_impl::experimental.") -- add_definitions( -- -DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc_impl::experimental) -+ LINK_LIBRARIES gRPC::grpc gRPC::grpc++ -+ OUTPUT_VARIABLE TLS_CREDENTIALS_OPTIONS_CHECK_OUTPUT CXX_STANDARD 11) -+ message(STATUS "${TLS_CREDENTIALS_OPTIONS_CHECK_OUTPUT}") -+ if(HAS_GRPC_VERSION) -+ set(${DST_VAR} "${DETECT_VERSION}" PARENT_SCOPE) - else() -- message(STATUS "TlsCredentialsOptions not found in grpc_impl::experimental.") -+ message( -+ STATUS -+ "TlsCredentialsOptions (for gRPC ${DETECT_VERSION}) not found in grpc::experimental." -+ ) - message(DEBUG "Build output:") -- list(APPEND CMAKE_MESSAGE_INDENT "check_tls_opts_127.cc: ") -- message(DEBUG ${TSL_CREDENTIALS_OPTIONS_CHECK_OUTPUT}) -+ list(APPEND CMAKE_MESSAGE_INDENT "${TEST_FILE}: ") -+ message(DEBUG ${TLS_CREDENTIALS_OPTIONS_CHECK_OUTPUT}) - list(REMOVE_AT CMAKE_MESSAGE_INDENT -1) - endif() - endif() -+endfunction() -+ -+test_grpc_version(GRPC_VERSION "1.36" "check_tls_opts_136.cc") -+test_grpc_version(GRPC_VERSION "1.34" "check_tls_opts_134.cc") -+test_grpc_version(GRPC_VERSION "1.32" "check_tls_opts_132.cc") -+test_grpc_version(GRPC_VERSION "1.27" "check_tls_opts_127.cc") -+message( -+ STATUS -+ "Found approximate gRPC version: ${GRPC_VERSION} (ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=${ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS})" -+ ) -+if(GRPC_VERSION EQUAL "1.27") -+ add_definitions(-DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc_impl::experimental) -+elseif(GRPC_VERSION EQUAL "1.32") -+ add_definitions(-DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) -+elseif(GRPC_VERSION EQUAL "1.34") -+ add_definitions(-DGRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS -+ -DGRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS_ROOT_CERTS -+ -DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) -+elseif(GRPC_VERSION EQUAL "1.36") -+ add_definitions(-DGRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS -+ -DGRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS=grpc::experimental) -+else() -+ message( -+ STATUS -+ "A proper version of gRPC could not be found to support TlsCredentialsOptions in Arrow Flight." -+ ) -+ message( -+ STATUS -+ "You may need a newer version of gRPC (>= 1.27), or the gRPC API has changed and Flight must be updated to match." -+ ) -+ if(ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS) -+ message( -+ FATAL_ERROR "Halting build since ARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS is set." -+ ) -+ endif() - endif() - - # Restore the CXXFLAGS that were modified above -diff --git a/cpp/src/arrow/flight/client.cc b/cpp/src/arrow/flight/client.cc -index 724f999fabe..cd6f3a97e58 100644 ---- a/cpp/src/arrow/flight/client.cc -+++ b/cpp/src/arrow/flight/client.cc -@@ -860,7 +860,7 @@ namespace { - // requires root CA certs, even if you are skipping server - // verification. - #if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) --constexpr char BLANK_ROOT_PEM[] = -+constexpr char kDummyRootCert[] = - "-----BEGIN CERTIFICATE-----\n" - "MIICwzCCAaugAwIBAgIJAM12DOkcaqrhMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV\n" - "BAMTCWxvY2FsaG9zdDAeFw0yMDEwMDcwODIyNDFaFw0zMDEwMDUwODIyNDFaMBQx\n" -@@ -893,11 +893,7 @@ class FlightClient::FlightClientImpl { - - if (scheme == kSchemeGrpcTls) { - if (options.disable_server_verification) { --#if !defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) -- return Status::NotImplemented( -- "Using encryption with server verification disabled is unsupported. " -- "Please use a release of Arrow Flight built with gRPC 1.27 or higher."); --#else -+#if defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) - namespace ge = GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS; - - // A callback to supply to TlsCredentialsOptions that accepts any server -@@ -910,16 +906,40 @@ class FlightClient::FlightClientImpl { - return 0; - } - }; -- -+ auto server_authorization_check = std::make_shared(); - noop_auth_check_ = std::make_shared( -- std::make_shared()); -+ server_authorization_check); -+#if defined(GRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS) -+ auto certificate_provider = -+ std::make_shared( -+ kDummyRootCert); -+#if defined(GRPC_USE_TLS_CHANNEL_CREDENTIALS_OPTIONS_ROOT_CERTS) -+ grpc::experimental::TlsChannelCredentialsOptions tls_options( -+ certificate_provider); -+#else -+ // While gRPC >= 1.36 does not require a root cert (it has a default) -+ // in practice the path it hardcodes is broken. See grpc/grpc#21655. -+ grpc::experimental::TlsChannelCredentialsOptions tls_options; -+ tls_options.set_certificate_provider(certificate_provider); -+#endif -+ tls_options.watch_root_certs(); -+ tls_options.set_root_cert_name("dummy"); -+ tls_options.set_server_verification_option( -+ grpc_tls_server_verification_option::GRPC_TLS_SKIP_ALL_SERVER_VERIFICATION); -+ tls_options.set_server_authorization_check_config(noop_auth_check_); -+#elif defined(GRPC_NAMESPACE_FOR_TLS_CREDENTIALS_OPTIONS) - auto materials_config = std::make_shared(); -- materials_config->set_pem_root_certs(BLANK_ROOT_PEM); -+ materials_config->set_pem_root_certs(kDummyRootCert); - ge::TlsCredentialsOptions tls_options( - GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE, - GRPC_TLS_SKIP_ALL_SERVER_VERIFICATION, materials_config, - std::shared_ptr(), noop_auth_check_); -+#endif - creds = ge::TlsCredentials(tls_options); -+#else -+ return Status::NotImplemented( -+ "Using encryption with server verification disabled is unsupported. " -+ "Please use a release of Arrow Flight built with gRPC 1.27 or higher."); - #endif - } else { - grpc::SslCredentialsOptions ssl_options; -diff --git a/cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc b/cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc -new file mode 100644 -index 00000000000..4ee2122ef57 ---- /dev/null -+++ b/cpp/src/arrow/flight/try_compile/check_tls_opts_134.cc -@@ -0,0 +1,44 @@ -+// Licensed to the Apache Software Foundation (ASF) under one -+// or more contributor license agreements. See the NOTICE file -+// distributed with this work for additional information -+// regarding copyright ownership. The ASF licenses this file -+// to you under the Apache License, Version 2.0 (the -+// "License"); you may not use this file except in compliance -+// with the License. You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, -+// software distributed under the License is distributed on an -+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -+// KIND, either express or implied. See the License for the -+// specific language governing permissions and limitations -+// under the License. -+ -+// Dummy file for checking if TlsCredentialsOptions exists in -+// the grpc::experimental namespace. gRPC starting from 1.34 -+// put it here. This is for supporting disabling server -+// validation when using TLS. -+ -+#include -+#include -+#include -+ -+// Dummy file for checking if TlsCredentialsOptions exists in -+// the grpc::experimental namespace. gRPC starting from 1.34 -+// puts it here. This is for supporting disabling server -+// validation when using TLS. -+ -+static void check() { -+ // In 1.34, there's no parameterless constructor; in 1.36, there's -+ // only a parameterless constructor -+ auto options = -+ std::make_shared(nullptr); -+ options->set_server_verification_option( -+ grpc_tls_server_verification_option::GRPC_TLS_SERVER_VERIFICATION); -+} -+ -+int main(int argc, const char** argv) { -+ check(); -+ return 0; -+} -diff --git a/cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc b/cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc -new file mode 100644 -index 00000000000..638eec67ba7 ---- /dev/null -+++ b/cpp/src/arrow/flight/try_compile/check_tls_opts_136.cc -@@ -0,0 +1,38 @@ -+// Licensed to the Apache Software Foundation (ASF) under one -+// or more contributor license agreements. See the NOTICE file -+// distributed with this work for additional information -+// regarding copyright ownership. The ASF licenses this file -+// to you under the Apache License, Version 2.0 (the -+// "License"); you may not use this file except in compliance -+// with the License. You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, -+// software distributed under the License is distributed on an -+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -+// KIND, either express or implied. See the License for the -+// specific language governing permissions and limitations -+// under the License. -+ -+// Dummy file for checking if TlsCredentialsOptions exists in -+// the grpc::experimental namespace. gRPC starting from 1.36 -+// puts it here. This is for supporting disabling server -+// validation when using TLS. -+ -+#include -+#include -+#include -+ -+static void check() { -+ // In 1.34, there's no parameterless constructor; in 1.36, there's -+ // only a parameterless constructor -+ auto options = std::make_shared(); -+ options->set_server_verification_option( -+ grpc_tls_server_verification_option::GRPC_TLS_SERVER_VERIFICATION); -+} -+ -+int main(int argc, const char** argv) { -+ check(); -+ return 0; -+} diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 11b3307523c..f3f7ca06c94 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -9,14 +9,10 @@ package: version: {{ version }} source: - url: https://dist.apache.org/repos/dist/release/arrow/arrow-{{ version }}/apache-arrow-{{ version }}.tar.gz - sha256: 73c2cc3be537aa1f3fd9490cfec185714168c9bfd599d23e287ab0cc0558e27a - patches: - - 0001-ARROW-11724-C-Resolve-namespace-collisions-with-prot.patch - - 9569.patch + path: ../../../../ build: - number: 10 + number: 0 # for cuda on win/linux, building with 9.2 is enough to be compatible with all later versions, # since arrow is only using libcuda, and not libcudart. skip: true # [(win or linux) and cuda_compiler_version not in ("None", "10.2")] From 0294889f4f5d4e6bc84c9c6a57312d27a20b47c7 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 8 Apr 2021 17:02:26 +0200 Subject: [PATCH 07/11] Update OSX builds --- ...sx_64_numpy1.17python3.6.____cpython.yaml} | 28 ++++---- ...sx_64_numpy1.17python3.7.____cpython.yaml} | 28 ++++---- ...sx_64_numpy1.17python3.8.____cpython.yaml} | 29 +++++---- ...sx_64_numpy1.19python3.9.____cpython.yaml} | 26 ++++---- .../osx_arm64_python3.8.____cpython.yaml | 65 +++++++++++++++++++ .../osx_arm64_python3.9.____cpython.yaml | 65 +++++++++++++++++++ dev/tasks/conda-recipes/azure.osx.yml | 8 ++- dev/tasks/tasks.yml | 26 ++++++-- 8 files changed, 212 insertions(+), 63 deletions(-) rename dev/tasks/conda-recipes/.ci_support/{osx_python3.6.____cpython.yaml => osx_64_numpy1.17python3.6.____cpython.yaml} (81%) rename dev/tasks/conda-recipes/.ci_support/{osx_python3.7.____cpython.yaml => osx_64_numpy1.17python3.7.____cpython.yaml} (81%) rename dev/tasks/conda-recipes/.ci_support/{osx_python3.8.____cpython.yaml => osx_64_numpy1.17python3.8.____cpython.yaml} (77%) rename dev/tasks/conda-recipes/.ci_support/{osx_python3.9.____cpython.yaml => osx_64_numpy1.19python3.9.____cpython.yaml} (82%) create mode 100644 dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml create mode 100644 dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml similarity index 81% rename from dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml index c019508dce4..7b2dbb34d76 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml @@ -1,15 +1,13 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - clang c_compiler_version: -- '10' +- '11' channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +17,24 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '10' +- '11' gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,16 +47,18 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - osx-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml similarity index 81% rename from dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml index 888071766a5..8e3e828ab8a 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml @@ -1,15 +1,13 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - clang c_compiler_version: -- '10' +- '11' channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +17,24 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '10' +- '11' gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.16' +- '1.17' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,16 +47,18 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - osx-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml similarity index 77% rename from dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml index ad449921253..cdd53c6006e 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml @@ -1,15 +1,13 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.54 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - clang c_compiler_version: -- '10' +- '11' channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +17,24 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '10' +- '11' gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.30' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.17.3' +- '1.17' orc: -- 1.6.4 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,11 +47,18 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.08.01 +- 2021.04.01 snappy: - '1' target_platform: - osx-64 +thrift_cpp: +- 0.14.1 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml similarity index 82% rename from dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml index c630217d111..37df6a9ec53 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_python3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml @@ -1,15 +1,13 @@ MACOSX_DEPLOYMENT_TARGET: - '10.9' aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: - clang c_compiler_version: -- '10' +- '11' channel_sources: - conda-forge,defaults channel_targets: @@ -19,26 +17,24 @@ cuda_compiler_version: cxx_compiler: - clangxx cxx_compiler_version: -- '10' +- '11' gflags: - '2.2' glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 macos_machine: - x86_64-apple-darwin13.4.0 numpy: - '1.19' orc: -- 1.6.5 +- 1.6.7 pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -51,16 +47,18 @@ pin_run_as_build: python: - 3.9.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - osx-64 +thrift_cpp: +- 0.14.1 zip_keys: - - c_compiler_version - cxx_compiler_version -- - numpy - - python +- - python + - numpy zlib: - '1.2' zstd: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml new file mode 100644 index 00000000000..5894b8ee70b --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +aws_sdk_cpp: +- 1.8.151 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge/label/rust_dev,conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.36' +libprotobuf: +- '3.15' +lz4_c: +- 1.9.3 +macos_machine: +- arm64-apple-darwin20.0.0 +numpy: +- '1.19' +orc: +- 1.6.7 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.8.* *_cpython +re2: +- 2021.04.01 +snappy: +- '1' +target_platform: +- osx-arm64 +thrift_cpp: +- 0.14.1 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml new file mode 100644 index 00000000000..4e6014c5db8 --- /dev/null +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +aws_sdk_cpp: +- 1.8.151 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge/label/rust_dev,conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- 0.4.0 +grpc_cpp: +- '1.36' +libprotobuf: +- '3.15' +lz4_c: +- 1.9.3 +macos_machine: +- arm64-apple-darwin20.0.0 +numpy: +- '1.19' +orc: +- 1.6.7 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2021.04.01 +snappy: +- '1' +target_platform: +- osx-arm64 +thrift_cpp: +- 0.14.1 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.4' diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 58afa8045cb..9b428aaeb33 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -58,9 +58,13 @@ jobs: - script: | source activate base + if [[ "${CONFIG}" == osx_arm* ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" + fi conda build arrow-cpp parquet-cpp \ -m ./.ci_support/${CONFIG}.yaml \ --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \ + ${EXTRA_CB_OPTIONS:-} \ --output-folder ./build_artifacts if [ ! -z "${R_CONFIG}" ]; then @@ -71,5 +75,5 @@ jobs: workingDirectory: arrow/dev/tasks/conda-recipes displayName: Build recipes - {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-64/*.tar.bz2") }} - {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-64/*.tar.bz2") }} + {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} + {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 9502b546dcd..6bf1bd85872 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -278,7 +278,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_python3.6.____cpython + config: osx_64_numpy1.17python3.6.____cpython r_config: osx_64_r_base3.6 artifacts: - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -288,7 +288,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_python3.7.____cpython + config: osx_64_numpy1.17python3.7.____cpython r_config: osx_64_r_base4.0 artifacts: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -298,7 +298,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_python3.8.____cpython + config: osx_64_numpy1.17python3.8.____cpython artifacts: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -307,7 +307,25 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_python3.9.____cpython + config: osx_64_numpy1.19python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-osx-arm64-clang-py38: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_arm64_python3.8.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-osx-arm64-clang-py39: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_arm64_python3.8.____cpython artifacts: - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 From e32cecb59d2a1622a0138ec7cf43a2b75df1b537 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 8 Apr 2021 17:10:50 +0200 Subject: [PATCH 08/11] Don't build parquet-cpp anymore --- dev/tasks/conda-recipes/azure.osx.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/tasks/conda-recipes/azure.osx.yml b/dev/tasks/conda-recipes/azure.osx.yml index 9b428aaeb33..dbb1a68aca6 100755 --- a/dev/tasks/conda-recipes/azure.osx.yml +++ b/dev/tasks/conda-recipes/azure.osx.yml @@ -58,10 +58,11 @@ jobs: - script: | source activate base + set +x if [[ "${CONFIG}" == osx_arm* ]]; then EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" fi - conda build arrow-cpp parquet-cpp \ + conda build arrow-cpp \ -m ./.ci_support/${CONFIG}.yaml \ --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \ ${EXTRA_CB_OPTIONS:-} \ From 44a000561497b37e2b15e0ccb52d1294145b8c3b Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 9 Apr 2021 09:38:29 +0200 Subject: [PATCH 09/11] Revert commented Windows commands --- .../conda-recipes/arrow-cpp/bld-pyarrow.bat | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat index 23a09149dad..ee60f8cec90 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -1,16 +1,17 @@ +@echo on pushd "%SRC_DIR%"\python @rem the symlinks for cmake modules don't work here -del cmake_modules\BuildUtils.cmake -del cmake_modules\SetupCxxFlags.cmake -del cmake_modules\CompilerInfo.cmake -del cmake_modules\FindNumPy.cmake -del cmake_modules\FindPythonLibsNew.cmake -copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ -copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ -copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ -copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ -copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ +@rem del cmake_modules\BuildUtils.cmake +@rem del cmake_modules\SetupCxxFlags.cmake +@rem del cmake_modules\CompilerInfo.cmake +@rem del cmake_modules\FindNumPy.cmake +@rem del cmake_modules\FindPythonLibsNew.cmake +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ SET ARROW_HOME=%LIBRARY_PREFIX% SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% From f337001689364c365389121d19d916acee841ae5 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Fri, 9 Apr 2021 12:04:59 +0200 Subject: [PATCH 10/11] Update Windows variants --- ...onNonenumpy1.17python3.6.____cpython.yaml} | 20 +++++++++---------- ...onNonenumpy1.17python3.7.____cpython.yaml} | 20 +++++++++---------- ...onNonenumpy1.17python3.8.____cpython.yaml} | 20 +++++++++---------- ...onNonenumpy1.19python3.9.____cpython.yaml} | 18 ++++++++--------- dev/tasks/tasks.yml | 17 ++++++++++++---- 5 files changed, 52 insertions(+), 43 deletions(-) rename dev/tasks/conda-recipes/.ci_support/{win_python3.6.____cpython.yaml => win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml} (82%) rename dev/tasks/conda-recipes/.ci_support/{win_python3.7.____cpython.yaml => win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml} (82%) rename dev/tasks/conda-recipes/.ci_support/{win_python3.8.____cpython.yaml => win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml} (82%) rename dev/tasks/conda-recipes/.ci_support/{win_python3.9.____cpython.yaml => win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml} (83%) diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml similarity index 82% rename from dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml index 40199c679de..8fbbb64af9c 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.6.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml @@ -1,7 +1,5 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: @@ -10,6 +8,8 @@ channel_sources: - conda-forge,defaults channel_targets: - conda-forge main +cuda_compiler: +- nvcc cuda_compiler_version: - None cxx_compiler: @@ -19,16 +19,14 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -41,11 +39,13 @@ pin_run_as_build: python: - 3.6.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - win-64 +thrift_cpp: +- 0.14.1 zip_keys: - - numpy - python diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml similarity index 82% rename from dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml index 88d17108a98..4b702a38980 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.7.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml @@ -1,7 +1,5 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: @@ -10,6 +8,8 @@ channel_sources: - conda-forge,defaults channel_targets: - conda-forge main +cuda_compiler: +- nvcc cuda_compiler_version: - None cxx_compiler: @@ -19,16 +19,14 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -41,11 +39,13 @@ pin_run_as_build: python: - 3.7.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - win-64 +thrift_cpp: +- 0.14.1 zip_keys: - - numpy - python diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml similarity index 82% rename from dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml index e18785de135..6ae6c2fde4f 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.8.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml @@ -1,7 +1,5 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: @@ -10,6 +8,8 @@ channel_sources: - conda-forge,defaults channel_targets: - conda-forge main +cuda_compiler: +- nvcc cuda_compiler_version: - None cxx_compiler: @@ -19,16 +19,14 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: -- '1.16' +- '1.17' pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -41,11 +39,13 @@ pin_run_as_build: python: - 3.8.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - win-64 +thrift_cpp: +- 0.14.1 zip_keys: - - numpy - python diff --git a/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml similarity index 83% rename from dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml rename to dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml index 6177f96ce01..73a8b5099bb 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_python3.9.____cpython.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml @@ -1,7 +1,5 @@ aws_sdk_cpp: -- 1.8.63 -boost_cpp: -- 1.74.0 +- 1.8.151 bzip2: - '1' c_compiler: @@ -10,6 +8,8 @@ channel_sources: - conda-forge,defaults channel_targets: - conda-forge main +cuda_compiler: +- nvcc cuda_compiler_version: - None cxx_compiler: @@ -19,16 +19,14 @@ gflags: glog: - 0.4.0 grpc_cpp: -- '1.32' +- '1.36' libprotobuf: -- '3.13' +- '3.15' lz4_c: -- 1.9.2 +- 1.9.3 numpy: - '1.19' pin_run_as_build: - boost-cpp: - max_pin: x.x.x bzip2: max_pin: x lz4-c: @@ -41,11 +39,13 @@ pin_run_as_build: python: - 3.9.* *_cpython re2: -- 2020.10.01 +- 2021.04.01 snappy: - '1' target_platform: - win-64 +thrift_cpp: +- 0.14.1 zip_keys: - - numpy - python diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 6bf1bd85872..561794146ab 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -325,7 +325,7 @@ tasks: ci: azure template: conda-recipes/azure.osx.yml params: - config: osx_arm64_python3.8.____cpython + config: osx_arm64_python3.9.____cpython artifacts: - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -336,7 +336,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_python3.6.____cpython + config: win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython r_config: win_64_r_base3.6 artifacts: - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -346,7 +346,7 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_python3.7.____cpython + config: win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython r_config: win_64_r_base4.0 artifacts: - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 @@ -356,7 +356,16 @@ tasks: ci: azure template: conda-recipes/azure.win.yml params: - config: win_python3.8.____cpython + config: win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-win-vs2017-py39: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython artifacts: - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 From a60a8db71b5f868e871269b69a46fa135e94e3d2 Mon Sep 17 00:00:00 2001 From: "Uwe L. Korn" Date: Thu, 15 Apr 2021 12:17:47 +0200 Subject: [PATCH 11/11] Add comments about difference to conda-forge --- dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat | 1 + dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat index ee60f8cec90..89cec3710c3 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat +++ b/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -2,6 +2,7 @@ pushd "%SRC_DIR%"\python @rem the symlinks for cmake modules don't work here +@rem NOTE: In contrast to conda-forge, they work here as we clone from git. @rem del cmake_modules\BuildUtils.cmake @rem del cmake_modules\SetupCxxFlags.cmake @rem del cmake_modules\CompilerInfo.cmake diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index f3f7ca06c94..48a8629866d 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -1,3 +1,4 @@ +# NOTE: In constrast to the conda-forge recipe, ARROW_VERSION is a templated variable here. {% set version = ARROW_VERSION %} {% set cuda_enabled = cuda_compiler_version != "None" %} {% set build_ext_version = ARROW_VERSION %}