From 3dab2baf9b40757fc6d84495ea72750d084b3a1f Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Fri, 28 Jul 2023 10:19:57 +0100 Subject: [PATCH 1/7] Install azurite --- .github/workflows/cpp.yml | 3 +++ ci/docker/ubuntu-20.04-cpp.dockerfile | 4 +++ ci/docker/ubuntu-22.04-cpp.dockerfile | 4 +++ ci/scripts/install_azurite.sh | 37 +++++++++++++++++++++++++++ cpp/Brewfile | 1 + 5 files changed, 49 insertions(+) create mode 100755 ci/scripts/install_azurite.sh diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index cd12be11488..7c5a066877d 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -424,6 +424,9 @@ jobs: run: | ci/scripts/install_gcs_testbench.sh default echo "PYTHON_BIN_DIR=$(cygpath --windows $(dirname $(which python3.exe)))" >> $GITHUB_ENV + - name: Install Azurite Storage Emulator + shell: bash + run: ci/scripts/install_azurite.sh - name: Test shell: msys2 {0} run: | diff --git a/ci/docker/ubuntu-20.04-cpp.dockerfile b/ci/docker/ubuntu-20.04-cpp.dockerfile index f94494177e8..125f1f48d48 100644 --- a/ci/docker/ubuntu-20.04-cpp.dockerfile +++ b/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -103,6 +103,7 @@ RUN apt-get update -y -q && \ make \ ninja-build \ nlohmann-json3-dev \ + npm \ pkg-config \ protobuf-compiler \ python3-dev \ @@ -123,6 +124,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default +COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_azurite.sh + COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_ceph.sh diff --git a/ci/docker/ubuntu-22.04-cpp.dockerfile b/ci/docker/ubuntu-22.04-cpp.dockerfile index e773c6f1ee6..0840b3fa5c6 100644 --- a/ci/docker/ubuntu-22.04-cpp.dockerfile +++ b/ci/docker/ubuntu-22.04-cpp.dockerfile @@ -102,6 +102,7 @@ RUN apt-get update -y -q && \ make \ ninja-build \ nlohmann-json3-dev \ + npm \ pkg-config \ protobuf-compiler \ protobuf-compiler-grpc \ @@ -153,6 +154,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_gcs_testbench.sh default +COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_azurite.sh + COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin diff --git a/ci/scripts/install_azurite.sh b/ci/scripts/install_azurite.sh new file mode 100755 index 00000000000..2e7008360fd --- /dev/null +++ b/ci/scripts/install_azurite.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +case "$(uname)" in + Darwin) + npm install -g azurite + which azurite + ;; + MINGW*) + choco install nodejs.install + npm install -g azurite + ;; + Linux) + npm install -g azurite + which azurite + ;; +esac +echo "node version = $(node --version)" +echo "azurite version = $(azurite --version)" \ No newline at end of file diff --git a/cpp/Brewfile b/cpp/Brewfile index 580e8d3f115..58015d2121b 100644 --- a/cpp/Brewfile +++ b/cpp/Brewfile @@ -30,6 +30,7 @@ brew "grpc" brew "llvm@14" brew "lz4" brew "ninja" +brew "node" brew "openssl@3" brew "protobuf" brew "python" From 101f81affe295c62ae5533f593f0d347be2ee4b7 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 1 Aug 2023 22:48:00 +0100 Subject: [PATCH 2/7] Add C++ `AzuriteEnv` --- cpp/src/arrow/filesystem/azurefs_test.cc | 65 +++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index 0f03e88393a..10cc89f764a 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -16,6 +16,8 @@ // under the License. #include "arrow/filesystem/azurefs.h" +#include +#include "arrow/util/io_util.h" #include #include @@ -27,15 +29,76 @@ #include "arrow/testing/util.h" namespace arrow { +using internal::TemporaryDir; namespace fs { namespace { +namespace bp = boost::process; using ::testing::IsEmpty; using ::testing::Not; using ::testing::NotNull; -// Placeholder test for file structure +class AzuriteEnv : public ::testing::Environment { + public: + AzuriteEnv() { + account_name_ = "devstoreaccount1"; + account_key_ = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/" + "KBHBeksoGMGw=="; + auto exe_path = bp::search_path("azurite"); + if (exe_path.empty()) { + auto error = std::string("Could not find Azurite emulator."); + status_ = Status::Invalid(error); + return; + } + auto temp_dir_ = TemporaryDir::Make("azurefs-test-").ValueOrDie(); + server_process_ = bp::child(boost::this_process::environment(), exe_path, "--silent", + "--location", temp_dir_->path().ToString(), "--debug", + temp_dir_->path().ToString() + "/debug.log"); + if (!(server_process_.valid() && server_process_.running())) { + auto error = "Could not start Azurite emulator."; + server_process_.terminate(); + server_process_.wait(); + status_ = Status::Invalid(error); + return; + } + status_ = Status::OK(); + } + + ~AzuriteEnv() override { + server_process_.terminate(); + server_process_.wait(); + } + + const std::string& account_name() const { return account_name_; } + const std::string& account_key() const { return account_key_; } + const Status status() const { return status_; } + + private: + std::string account_name_; + std::string account_key_; + bp::child server_process_; + Status status_; + std::unique_ptr temp_dir_; +}; + +auto* azurite_env = ::testing::AddGlobalTestEnvironment(new AzuriteEnv); + +AzuriteEnv* GetAzuriteEnv() { + return ::arrow::internal::checked_cast(azurite_env); +} + +// Placeholder tests for file structure // TODO: GH-18014 Remove once a proper test is added +TEST(AzureFileSystem, InitialiseAzurite) { + const std::string& account_name = GetAzuriteEnv()->account_name(); + const std::string& account_key = GetAzuriteEnv()->account_key(); + EXPECT_EQ(account_name, "devstoreaccount1"); + EXPECT_EQ(account_key, + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw=="); +} + TEST(AzureFileSystem, OptionsCompare) { AzureOptions options; EXPECT_TRUE(options.Equals(options)); From 1604ed9afcd5ab33d0b31b680e3980cedcb9b461 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 1 Aug 2023 23:10:31 +0100 Subject: [PATCH 3/7] Correct mac os azurite install --- .github/workflows/cpp.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 7c5a066877d..2e58198668b 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -201,6 +201,8 @@ jobs: ci/scripts/install_minio.sh latest /usr/local - name: Install Google Cloud Storage Testbench run: ci/scripts/install_gcs_testbench.sh default + - name: Install Azurite Storage Emulator + run: ci/scripts/install_azurite.sh - name: Setup ccache run: | ci/scripts/ccache_setup.sh @@ -424,9 +426,6 @@ jobs: run: | ci/scripts/install_gcs_testbench.sh default echo "PYTHON_BIN_DIR=$(cygpath --windows $(dirname $(which python3.exe)))" >> $GITHUB_ENV - - name: Install Azurite Storage Emulator - shell: bash - run: ci/scripts/install_azurite.sh - name: Test shell: msys2 {0} run: | From 3936dc25eb251aad2e2946da6dd865f1d66428f0 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 1 Aug 2023 23:14:09 +0100 Subject: [PATCH 4/7] Add the install azurite script to the workflow trigger paths --- .github/workflows/cpp.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 2e58198668b..eaccf25403e 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -23,6 +23,7 @@ on: - '.github/workflows/cpp.yml' - 'ci/docker/**' - 'ci/scripts/cpp_*' + - 'ci/scripts/install_azurite.sh' - 'ci/scripts/install_gcs_testbench.sh' - 'ci/scripts/install_minio.sh' - 'ci/scripts/msys2_*' @@ -34,6 +35,7 @@ on: - '.github/workflows/cpp.yml' - 'ci/docker/**' - 'ci/scripts/cpp_*' + - 'ci/scripts/install_azurite.sh' - 'ci/scripts/install_gcs_testbench.sh' - 'ci/scripts/install_minio.sh' - 'ci/scripts/msys2_*' From d1102fcb1ad7b7390137b4c24e546fe5a98751ef Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Tue, 1 Aug 2023 23:53:14 +0100 Subject: [PATCH 5/7] Lint --- cpp/src/arrow/filesystem/azurefs_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index 10cc89f764a..b234324312c 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -95,8 +95,8 @@ TEST(AzureFileSystem, InitialiseAzurite) { const std::string& account_key = GetAzuriteEnv()->account_key(); EXPECT_EQ(account_name, "devstoreaccount1"); EXPECT_EQ(account_key, - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" - "K1SZFPTOtr/KBHBeksoGMGw=="); + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw=="); } TEST(AzureFileSystem, OptionsCompare) { From b5a4c0431bc0779d2bd848c17e3279e831e9888d Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Mon, 7 Aug 2023 09:00:08 +0100 Subject: [PATCH 6/7] Update cpp/src/arrow/filesystem/azurefs_test.cc Co-authored-by: Sutou Kouhei --- cpp/src/arrow/filesystem/azurefs_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index b234324312c..9ee7db4d785 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -51,7 +51,7 @@ class AzuriteEnv : public ::testing::Environment { status_ = Status::Invalid(error); return; } - auto temp_dir_ = TemporaryDir::Make("azurefs-test-").ValueOrDie(); + auto temp_dir_ = *TemporaryDir::Make("azurefs-test-"); server_process_ = bp::child(boost::this_process::environment(), exe_path, "--silent", "--location", temp_dir_->path().ToString(), "--debug", temp_dir_->path().ToString() + "/debug.log"); From 651438ecf145cf6bf7e8e3ba439e6c5a3c355132 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Mon, 7 Aug 2023 09:02:50 +0100 Subject: [PATCH 7/7] Updated boost/process include --- cpp/src/arrow/filesystem/azurefs_test.cc | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index 9ee7db4d785..e940c5bd1bc 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -15,8 +15,25 @@ // specific language governing permissions and limitations // under the License. -#include "arrow/filesystem/azurefs.h" +#include // Missing include in boost/process + +// This boost/asio/io_context.hpp include is needless for no MinGW +// build. +// +// This is for including boost/asio/detail/socket_types.hpp before any +// "#include ". boost/asio/detail/socket_types.hpp doesn't +// work if windows.h is already included. boost/process.h -> +// boost/process/args.hpp -> boost/process/detail/basic_cmd.hpp +// includes windows.h. boost/process/args.hpp is included before +// boost/process/async.h that includes +// boost/asio/detail/socket_types.hpp implicitly is included. +#include +// We need BOOST_USE_WINDOWS_H definition with MinGW when we use +// boost/process.hpp. See BOOST_USE_WINDOWS_H=1 in +// cpp/cmake_modules/ThirdpartyToolchain.cmake for details. #include + +#include "arrow/filesystem/azurefs.h" #include "arrow/util/io_util.h" #include