From 848612bb0c2afa8cdd3d0885fa2c74a3fa409ae4 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Fri, 3 Dec 2021 18:42:56 +0000 Subject: [PATCH 1/3] ARROW-14506: [C++] Conda support for google-cloud-cpp This PR adds support for `google-cloud-cpp` to the Conda files. Probably the most difficult change to grok is the change to compile with C++17 when using Conda: - Conda defaults all its builds to C++17, [this bug](https://github.com/conda/conda-build/issues/3375) goes into some detail as to why. - Arrow defaults to C++11 if no `CMAKE_CXX_STANDARD` argument is provided. - Abseil's ABI changes when used from C++11 vs. C++17, see https://github.com/abseil/abseil-cpp/issues/696 - Therefore, one must compile with C++17 to use Abseil in Conda. - And because `google-cloud-cpp` has a direct dependency on Abseil, exposed through the headers, one must use C++17 to use `google-cloud-cpp` too. --- ci/conda_env_cpp.txt | 4 ++++ ci/docker/conda-cpp.dockerfile | 9 ++++++--- ci/scripts/cpp_build.sh | 1 + dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 6d1ebf35341..49c3142e64c 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -25,6 +25,7 @@ cmake gflags glog gmock>=1.10.0 +google-cloud-cpp>=1.34.0 grpc-cpp>=1.27.3 gtest>=1.10.0 libprotobuf @@ -32,6 +33,9 @@ libutf8proc lz4-c make ninja +# Required by google-cloud-cpp, the Conda package is missing the dependency: +# https://github.com/conda-forge/google-cloud-cpp-feedstock/issues/28 +nlohmann_json pkg-config python rapidjson diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index 8fd5e46fd6d..6395c698ed4 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -22,9 +22,6 @@ FROM ${repo}:${arch}-conda COPY ci/scripts/install_minio.sh /arrow/ci/scripts RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda -COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts -RUN /arrow/ci/scripts/install_gcs_testbench.sh default - # install the required conda packages into the test environment COPY ci/conda_env_cpp.txt \ ci/conda_env_gandiva.txt \ @@ -37,11 +34,16 @@ RUN mamba install \ valgrind && \ mamba clean --all +# We want to install the GCS testbench using the same Python binary that the Conda code will use. +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts +RUN /arrow/ci/scripts/install_gcs_testbench.sh default + ENV ARROW_BUILD_TESTS=ON \ ARROW_DATASET=ON \ ARROW_DEPENDENCY_SOURCE=CONDA \ ARROW_FLIGHT=ON \ ARROW_GANDIVA=ON \ + ARROW_GCS=ON \ ARROW_HOME=$CONDA_PREFIX \ ARROW_ORC=ON \ ARROW_PARQUET=ON \ @@ -54,6 +56,7 @@ ENV ARROW_BUILD_TESTS=ON \ ARROW_WITH_SNAPPY=ON \ ARROW_WITH_ZLIB=ON \ ARROW_WITH_ZSTD=ON \ + CMAKE_CXX_STANDARD=17 \ GTest_SOURCE=BUNDLED \ PARQUET_BUILD_EXAMPLES=ON \ PARQUET_BUILD_EXECUTABLES=ON \ diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh index f791ddd5645..77a16edfbec 100755 --- a/ci/scripts/cpp_build.sh +++ b/ci/scripts/cpp_build.sh @@ -119,6 +119,7 @@ cmake \ -DCMAKE_BUILD_TYPE=${ARROW_BUILD_TYPE:-debug} \ -DCMAKE_C_FLAGS="${CFLAGS:-}" \ -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \ + -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-11}" \ -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \ -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \ -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \ diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index 9e4c02c5c6e..814c8d72c1b 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -58,6 +58,7 @@ cmake \ -DARROW_DEPENDENCY_SOURCE=SYSTEM \ -DARROW_FLIGHT=ON \ -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=ON \ + -DARROW_GCS=ON \ -DARROW_HDFS=ON \ -DARROW_JEMALLOC=ON \ -DARROW_MIMALLOC=ON \ From c28edda4f598d98faadb638218649e6ac7834ba4 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Fri, 3 Dec 2021 20:26:41 +0000 Subject: [PATCH 2/3] Avoid breaking Windows --- ci/conda_env_cpp.txt | 4 ---- ci/conda_env_cpp_linux.txt | 21 +++++++++++++++++++++ ci/docker/conda-cpp.dockerfile | 2 ++ 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 ci/conda_env_cpp_linux.txt diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt index 49c3142e64c..6d1ebf35341 100644 --- a/ci/conda_env_cpp.txt +++ b/ci/conda_env_cpp.txt @@ -25,7 +25,6 @@ cmake gflags glog gmock>=1.10.0 -google-cloud-cpp>=1.34.0 grpc-cpp>=1.27.3 gtest>=1.10.0 libprotobuf @@ -33,9 +32,6 @@ libutf8proc lz4-c make ninja -# Required by google-cloud-cpp, the Conda package is missing the dependency: -# https://github.com/conda-forge/google-cloud-cpp-feedstock/issues/28 -nlohmann_json pkg-config python rapidjson diff --git a/ci/conda_env_cpp_linux.txt b/ci/conda_env_cpp_linux.txt new file mode 100644 index 00000000000..44573366db9 --- /dev/null +++ b/ci/conda_env_cpp_linux.txt @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +google-cloud-cpp>=1.34.0 +# Required by google-cloud-cpp, the Conda package is missing the dependency: +# https://github.com/conda-forge/google-cloud-cpp-feedstock/issues/28 +nlohmann_json diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile index 6395c698ed4..5af5612ee10 100644 --- a/ci/docker/conda-cpp.dockerfile +++ b/ci/docker/conda-cpp.dockerfile @@ -24,10 +24,12 @@ RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda # install the required conda packages into the test environment COPY ci/conda_env_cpp.txt \ + ci/conda_env_cpp_linux.txt \ ci/conda_env_gandiva.txt \ /arrow/ci/ RUN mamba install \ --file arrow/ci/conda_env_cpp.txt \ + --file arrow/ci/conda_env_cpp_linux.txt \ --file arrow/ci/conda_env_gandiva.txt \ compilers \ doxygen \ From da0e64a16ab4f1be4cfa9c00abc422c597c6c684 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Mon, 6 Dec 2021 14:49:48 +0000 Subject: [PATCH 3/3] Address review comments --- ci/conda_env_cpp_linux.txt | 3 +++ dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 1 + 2 files changed, 4 insertions(+) diff --git a/ci/conda_env_cpp_linux.txt b/ci/conda_env_cpp_linux.txt index 44573366db9..d51271f09db 100644 --- a/ci/conda_env_cpp_linux.txt +++ b/ci/conda_env_cpp_linux.txt @@ -15,7 +15,10 @@ # specific language governing permissions and limitations # under the License. +# The Conda packages for `google-cloud-cpp` lack Windows support: +# https://github.com/conda-forge/google-cloud-cpp-feedstock/issues/76 google-cloud-cpp>=1.34.0 + # Required by google-cloud-cpp, the Conda package is missing the dependency: # https://github.com/conda-forge/google-cloud-cpp-feedstock/issues/28 nlohmann_json diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index 48a8629866d..44913c44543 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -73,6 +73,7 @@ outputs: - c-ares - gflags - glog + - google-cloud-cpp >=1.34.0 - grpc-cpp - libprotobuf - clangdev 10 # [not (osx and arm64)]