diff --git a/ci/docker/conda.dockerfile b/ci/docker/conda.dockerfile index 2e773b5437e..13bf902e9d9 100644 --- a/ci/docker/conda.dockerfile +++ b/ci/docker/conda.dockerfile @@ -36,6 +36,8 @@ COPY ci/scripts/install_conda.sh \ /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix} RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix} +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts +RUN /arrow/ci/scripts/install_gcs_testbench.sh default # create a conda environment ADD ci/conda_env_unix.txt /arrow/ci/ diff --git a/ci/docker/debian-10-cpp.dockerfile b/ci/docker/debian-10-cpp.dockerfile index d99a2c161bd..d4a78707bf8 100644 --- a/ci/docker/debian-10-cpp.dockerfile +++ b/ci/docker/debian-10-cpp.dockerfile @@ -65,15 +65,17 @@ RUN apt-get update -y -q && \ ninja-build \ pkg-config \ protobuf-compiler \ + python3-pip \ rapidjson-dev \ tzdata \ zlib1g-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default ENV ARROW_BUILD_TESTS=ON \ ARROW_DATASET=ON \ diff --git a/ci/docker/debian-11-cpp.dockerfile b/ci/docker/debian-11-cpp.dockerfile index 96c1a1738f7..b809c09c99f 100644 --- a/ci/docker/debian-11-cpp.dockerfile +++ b/ci/docker/debian-11-cpp.dockerfile @@ -63,15 +63,17 @@ RUN apt-get update -y -q && \ ninja-build \ pkg-config \ protobuf-compiler-grpc \ + python3-pip \ rapidjson-dev \ tzdata \ zlib1g-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default ENV ARROW_BUILD_TESTS=ON \ ARROW_DATASET=ON \ diff --git a/ci/docker/fedora-33-cpp.dockerfile b/ci/docker/fedora-33-cpp.dockerfile index 9dde6999510..5de210b3fa5 100644 --- a/ci/docker/fedora-33-cpp.dockerfile +++ b/ci/docker/fedora-33-cpp.dockerfile @@ -53,6 +53,7 @@ RUN dnf update -y && \ openssl-devel \ protobuf-devel \ python \ + python-pip \ rapidjson-devel \ re2-devel \ snappy-devel \ @@ -62,9 +63,10 @@ RUN dnf update -y && \ which \ zlib-devel -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default ENV ARROW_BUILD_TESTS=ON \ ARROW_DEPENDENCY_SOURCE=SYSTEM \ diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 3a84a11c9b6..fe0a5212b93 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -85,9 +85,10 @@ COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ COPY r/DESCRIPTION /arrow/r/ RUN /arrow/ci/scripts/r_deps.sh /arrow -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default # Set up Python 3 and its dependencies RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile index a501d69955c..568b90c227f 100644 --- a/ci/docker/linux-r.dockerfile +++ b/ci/docker/linux-r.dockerfile @@ -40,6 +40,7 @@ ENV PATH "${RPREFIX}/bin:${PATH}" COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/ COPY ci/etc/rprofile /arrow/ci/etc/ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/r_docker_configure.sh COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index 35658b28a86..577b2d48c6b 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -88,15 +88,17 @@ RUN apt-get update -y -q && \ ninja-build \ pkg-config \ protobuf-compiler \ + python3-pip \ rapidjson-dev \ tzdata \ wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists* -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default # Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages diff --git a/ci/docker/ubuntu-20.10-cpp.dockerfile b/ci/docker/ubuntu-20.10-cpp.dockerfile index 6cefecfd678..f34005d6845 100644 --- a/ci/docker/ubuntu-20.10-cpp.dockerfile +++ b/ci/docker/ubuntu-20.10-cpp.dockerfile @@ -90,15 +90,17 @@ RUN apt-get update -y -q && \ pkg-config \ protobuf-compiler \ protobuf-compiler-grpc \ + python3-pip \ rapidjson-dev \ tzdata \ wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists* -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default # Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages diff --git a/ci/docker/ubuntu-21.04-cpp.dockerfile b/ci/docker/ubuntu-21.04-cpp.dockerfile index 18c377811bc..d6ec48334a6 100644 --- a/ci/docker/ubuntu-21.04-cpp.dockerfile +++ b/ci/docker/ubuntu-21.04-cpp.dockerfile @@ -88,15 +88,17 @@ RUN apt-get update -y -q && \ pkg-config \ protobuf-compiler \ protobuf-compiler-grpc \ + python3-pip \ rapidjson-dev \ tzdata \ wget && \ apt-get clean && \ rm -rf /var/lib/apt/lists* -COPY ci/scripts/install_minio.sh \ - /arrow/ci/scripts/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh default # Prioritize system packages and local installation # The following dependencies will be downloaded due to missing/invalid packages diff --git a/ci/scripts/install_gcs_testbench.sh b/ci/scripts/install_gcs_testbench.sh new file mode 100755 index 00000000000..d7a44906ebb --- /dev/null +++ b/ci/scripts/install_gcs_testbench.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +version=$1 +if [[ "${version}" -eq "default" ]]; then + version="v0.7.0" +fi + +pip install "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz" diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh index d138d030eca..befdcf77ab5 100755 --- a/ci/scripts/r_docker_configure.sh +++ b/ci/scripts/r_docker_configure.sh @@ -71,6 +71,10 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then if [ -f "/arrow/ci/scripts/install_minio.sh" ] && [ "`which wget`" ]; then /arrow/ci/scripts/install_minio.sh amd64 linux latest /usr/local fi + + if [ -f "/arrow/ci/scripts/install_gcs_testbench.sh" ] && [ "`which pip`" ]; then + /arrow/ci/scripts/install_gcs_testbench.sh default + fi fi # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786 diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index b82cccf24df..d7e433f4844 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -452,8 +452,8 @@ if(ARROW_FILESYSTEM) filesystem/util_internal.cc) if(ARROW_GCS) - list(APPEND ARROW_SRCS filesystem/gcsfs.cc) - set_source_files_properties(filesystem/gcsfs.cc + list(APPEND ARROW_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc) + set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc PROPERTIES SKIP_PRECOMPILE_HEADERS ON SKIP_UNITY_BUILD_INCLUSION ON) endif() diff --git a/cpp/src/arrow/filesystem/gcsfs.cc b/cpp/src/arrow/filesystem/gcsfs.cc index 58bbbbfd06c..ff911d02ab3 100644 --- a/cpp/src/arrow/filesystem/gcsfs.cc +++ b/cpp/src/arrow/filesystem/gcsfs.cc @@ -19,14 +19,46 @@ #include -#include - +#include "arrow/filesystem/gcsfs_internal.h" #include "arrow/filesystem/path_util.h" #include "arrow/result.h" #include "arrow/util/checked_cast.h" namespace arrow { namespace fs { +namespace { + +auto constexpr kSep = '/'; + +struct GcsPath { + std::string full_path; + std::string bucket; + std::string object; + + static Result FromString(const std::string& s) { + const auto src = internal::RemoveTrailingSlash(s); + auto const first_sep = src.find_first_of(kSep); + if (first_sep == 0) { + return Status::Invalid("Path cannot start with a separator ('", s, "')"); + } + if (first_sep == std::string::npos) { + return GcsPath{std::string(src), std::string(src), ""}; + } + GcsPath path; + path.full_path = std::string(src); + path.bucket = std::string(src.substr(0, first_sep)); + path.object = std::string(src.substr(first_sep + 1)); + return path; + } + + bool empty() const { return bucket.empty() && object.empty(); } + + bool operator==(const GcsPath& other) const { + return bucket == other.bucket && object == other.object; + } +}; + +} // namespace namespace gcs = google::cloud::storage; @@ -35,6 +67,13 @@ google::cloud::Options AsGoogleCloudOptions(const GcsOptions& o) { if (!o.endpoint_override.empty()) { std::string scheme = o.scheme; if (scheme.empty()) scheme = "https"; + if (scheme == "https") { + options.set( + google::cloud::MakeGoogleDefaultCredentials()); + } else { + options.set( + google::cloud::MakeInsecureCredentials()); + } options.set(scheme + "://" + o.endpoint_override); } return options; @@ -45,9 +84,31 @@ class GcsFileSystem::Impl { explicit Impl(GcsOptions o) : options_(std::move(o)), client_(AsGoogleCloudOptions(options_)) {} - GcsOptions const& options() const { return options_; } + const GcsOptions& options() const { return options_; } + + Result GetFileInfo(const GcsPath& path) { + if (!path.object.empty()) { + auto meta = client_.GetObjectMetadata(path.bucket, path.object); + return GetFileInfoImpl(path, std::move(meta).status(), FileType::File); + } + auto meta = client_.GetBucketMetadata(path.bucket); + return GetFileInfoImpl(path, std::move(meta).status(), FileType::Directory); + } private: + static Result GetFileInfoImpl(const GcsPath& path, + const google::cloud::Status& status, + FileType type) { + if (status.ok()) { + return FileInfo(path.full_path, type); + } + using ::google::cloud::StatusCode; + if (status.code() == StatusCode::kNotFound) { + return FileInfo(path.full_path, FileType::NotFound); + } + return internal::ToArrowStatus(status); + } + GcsOptions options_; gcs::Client client_; }; @@ -70,7 +131,8 @@ bool GcsFileSystem::Equals(const FileSystem& other) const { } Result GcsFileSystem::GetFileInfo(const std::string& path) { - return Status::NotImplemented("The GCS FileSystem is not fully implemented"); + ARROW_ASSIGN_OR_RAISE(auto p, GcsPath::FromString(path)); + return impl_->GetFileInfo(p); } Result GcsFileSystem::GetFileInfo(const FileSelector& select) { diff --git a/cpp/src/arrow/filesystem/gcsfs_internal.cc b/cpp/src/arrow/filesystem/gcsfs_internal.cc new file mode 100644 index 00000000000..22df5cebf67 --- /dev/null +++ b/cpp/src/arrow/filesystem/gcsfs_internal.cc @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/filesystem/gcsfs_internal.h" + +#include + +#include + +namespace arrow { +namespace fs { +namespace internal { + +Status ToArrowStatus(const google::cloud::Status& s) { + std::ostringstream os; + os << "google::cloud::Status(" << s << ")"; + switch (s.code()) { + case google::cloud::StatusCode::kOk: + break; + case google::cloud::StatusCode::kCancelled: + return Status::Cancelled(os.str()); + case google::cloud::StatusCode::kUnknown: + return Status::UnknownError(os.str()); + case google::cloud::StatusCode::kInvalidArgument: + return Status::Invalid(os.str()); + case google::cloud::StatusCode::kDeadlineExceeded: + return Status::IOError(os.str()); + case google::cloud::StatusCode::kNotFound: + // TODO: it is unclear if a better mapping would be possible. + return Status::UnknownError(os.str()); + case google::cloud::StatusCode::kAlreadyExists: + return Status::AlreadyExists(os.str()); + case google::cloud::StatusCode::kPermissionDenied: + case google::cloud::StatusCode::kUnauthenticated: + return Status::IOError(os.str()); + case google::cloud::StatusCode::kResourceExhausted: + return Status::CapacityError(os.str()); + case google::cloud::StatusCode::kFailedPrecondition: + case google::cloud::StatusCode::kAborted: + return Status::IOError(os.str()); + case google::cloud::StatusCode::kOutOfRange: + return Status::Invalid(os.str()); + case google::cloud::StatusCode::kUnimplemented: + return Status::NotImplemented(os.str()); + case google::cloud::StatusCode::kInternal: + case google::cloud::StatusCode::kUnavailable: + case google::cloud::StatusCode::kDataLoss: + return Status::IOError(os.str()); + } + return Status::OK(); +} + +} // namespace internal +} // namespace fs +} // namespace arrow diff --git a/cpp/src/arrow/filesystem/gcsfs_internal.h b/cpp/src/arrow/filesystem/gcsfs_internal.h new file mode 100644 index 00000000000..8d568701ed5 --- /dev/null +++ b/cpp/src/arrow/filesystem/gcsfs_internal.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "arrow/filesystem/filesystem.h" + +namespace arrow { +namespace fs { +namespace internal { + +Status ToArrowStatus(const google::cloud::Status& s); + +} // namespace internal +} // namespace fs +} // namespace arrow diff --git a/cpp/src/arrow/filesystem/gcsfs_test.cc b/cpp/src/arrow/filesystem/gcsfs_test.cc index 5d8c7b5a40a..0776872e3ac 100644 --- a/cpp/src/arrow/filesystem/gcsfs_test.cc +++ b/cpp/src/arrow/filesystem/gcsfs_test.cc @@ -19,21 +19,76 @@ #include #include +#include +#include +#include #include +#include #include -#include "arrow/testing/gtest_util.h" +#include "arrow/filesystem/gcsfs_internal.h" +#include "arrow/filesystem/test_util.h" #include "arrow/testing/util.h" namespace arrow { namespace fs { namespace { +namespace bp = boost::process; +namespace gc = google::cloud; +namespace gcs = google::cloud::storage; + +using ::testing::HasSubstr; using ::testing::IsEmpty; using ::testing::Not; using ::testing::NotNull; +auto const* kPreexistingBucket = "test-bucket-name"; + +class GcsIntegrationTest : public ::testing::Test { + public: + ~GcsIntegrationTest() override { + if (server_process_.valid()) { + // Brutal shutdown + server_process_.terminate(); + server_process_.wait(); + } + } + + protected: + void SetUp() override { + port_ = std::to_string(GetListenPort()); + auto exe_path = bp::search_path("python3"); + ASSERT_THAT(exe_path, Not(IsEmpty())); + + server_process_ = bp::child(boost::this_process::environment(), exe_path, "-m", + "testbench", "--port", port_); + + // Create a bucket in the testbench. This makes it easier to bootstrap GcsFileSystem + // and its tests. + auto client = gcs::Client( + google::cloud::Options{} + .set("http://127.0.0.1:" + port_) + .set(gc::MakeInsecureCredentials())); + google::cloud::StatusOr metadata = client.CreateBucketForProject( + kPreexistingBucket, "ignored-by-testbench", gcs::BucketMetadata{}); + ASSERT_TRUE(metadata.ok()) << "Failed to create bucket <" << kPreexistingBucket + << ">, status=" << metadata.status(); + } + + GcsOptions TestGcsOptions() { + GcsOptions options; + options.endpoint_override = "127.0.0.1:" + port_; + options.scheme = "http"; + return options; + } + + private: + std::string port_; + bp::child server_process_; +}; + TEST(GcsFileSystem, OptionsCompare) { GcsOptions a; GcsOptions b; @@ -45,20 +100,70 @@ TEST(GcsFileSystem, OptionsCompare) { EXPECT_FALSE(b.Equals(c)); } +TEST(GcsFileSystem, ToArrowStatusOK) { + Status actual = internal::ToArrowStatus(google::cloud::Status()); + EXPECT_TRUE(actual.ok()); +} + +TEST(GcsFileSystem, ToArrowStatus) { + struct { + google::cloud::StatusCode input; + arrow::StatusCode expected; + } cases[] = { + {google::cloud::StatusCode::kCancelled, StatusCode::Cancelled}, + {google::cloud::StatusCode::kUnknown, StatusCode::UnknownError}, + {google::cloud::StatusCode::kInvalidArgument, StatusCode::Invalid}, + {google::cloud::StatusCode::kDeadlineExceeded, StatusCode::IOError}, + {google::cloud::StatusCode::kNotFound, StatusCode::UnknownError}, + {google::cloud::StatusCode::kAlreadyExists, StatusCode::AlreadyExists}, + {google::cloud::StatusCode::kPermissionDenied, StatusCode::IOError}, + {google::cloud::StatusCode::kUnauthenticated, StatusCode::IOError}, + {google::cloud::StatusCode::kResourceExhausted, StatusCode::CapacityError}, + {google::cloud::StatusCode::kFailedPrecondition, StatusCode::IOError}, + {google::cloud::StatusCode::kAborted, StatusCode::IOError}, + {google::cloud::StatusCode::kOutOfRange, StatusCode::Invalid}, + {google::cloud::StatusCode::kUnimplemented, StatusCode::NotImplemented}, + {google::cloud::StatusCode::kInternal, StatusCode::IOError}, + {google::cloud::StatusCode::kUnavailable, StatusCode::IOError}, + {google::cloud::StatusCode::kDataLoss, StatusCode::IOError}, + }; + + for (const auto& test : cases) { + auto status = google::cloud::Status(test.input, "test-message"); + auto message = [&] { + std::ostringstream os; + os << status; + return os.str(); + }(); + SCOPED_TRACE("Testing with status=" + message); + const auto actual = arrow::fs::internal::ToArrowStatus(status); + EXPECT_EQ(actual.code(), test.expected); + EXPECT_THAT(actual.message(), HasSubstr(message)); + } +} + TEST(GcsFileSystem, FileSystemCompare) { - auto a = internal::MakeGcsFileSystemForTest(GcsOptions{}); + GcsOptions a_options; + a_options.scheme = "http"; + auto a = internal::MakeGcsFileSystemForTest(a_options); EXPECT_THAT(a, NotNull()); EXPECT_TRUE(a->Equals(*a)); - GcsOptions options; - options.endpoint_override = "localhost:1234"; - auto b = internal::MakeGcsFileSystemForTest(options); + GcsOptions b_options; + b_options.scheme = "http"; + b_options.endpoint_override = "localhost:1234"; + auto b = internal::MakeGcsFileSystemForTest(b_options); EXPECT_THAT(b, NotNull()); EXPECT_TRUE(b->Equals(*b)); EXPECT_FALSE(a->Equals(*b)); } +TEST_F(GcsIntegrationTest, MakeBucket) { + auto fs = internal::MakeGcsFileSystemForTest(TestGcsOptions()); + arrow::fs::AssertFileInfo(fs.get(), kPreexistingBucket, FileType::Directory); +} + } // namespace } // namespace fs } // namespace arrow diff --git a/docs/source/developers/docker.rst b/docs/source/developers/docker.rst index eaabad90df1..36b4687526e 100644 --- a/docs/source/developers/docker.rst +++ b/docs/source/developers/docker.rst @@ -210,6 +210,7 @@ responsible for. Like: - ``integration_pandas.sh``: execute the pandas integration tests. - ``install_minio.sh``: install minio server for multiple platforms. - ``install_conda.sh``: install miniconda for multiple platforms. +- ``install_gcs_testbench.sh``: install the GCS testbench for multiple platforms. The parametrization (like the C++ CMake options) is achieved via environment variables with useful defaults to keep the build configurations declarative.