From 7a5167c32b4778cf9a4c2e5d7c254ba8b644cb14 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Tue, 28 Feb 2023 19:40:50 +0800 Subject: [PATCH 01/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 29 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 6fde083581c92a..50c1107aaae9b2 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -41,20 +41,24 @@ static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; + // remove DISABLED_ when need run this test -#define TabletCooldownTest DISABLED_TabletCooldownTest +// #define TabletCooldownTest DISABLED_TabletCooldownTest class TabletCooldownTest : public testing::Test { public: + class S3ClientMock : public Aws::S3::S3Client { + S3ClientMock() {} + + S3ClientMock(const Aws::Auth::AWSCredentials &credentials, + const Aws::Client::ClientConfiguration &clientConfiguration, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy signPayloads, + bool use_virtual_addressing) + : Aws::S3::S3Client(credentials, clientConfiguration, signPayloads, + use_virtual_addressing) {} + }; + static void SetUpTestSuite() { - S3Conf s3_conf; - s3_conf.ak = config::test_s3_ak; - s3_conf.sk = config::test_s3_sk; - s3_conf.endpoint = config::test_s3_endpoint; - s3_conf.region = config::test_s3_region; - s3_conf.bucket = config::test_s3_bucket; - s3_conf.prefix = config::test_s3_prefix + "/tablet_cooldown_test"; - auto s3_fs = io::S3FileSystem::create(std::move(s3_conf), std::to_string(kResourceId)); - ASSERT_TRUE(s3_fs->connect().ok()); + S3ClientMock s3_fs; put_storage_resource(kResourceId, {s3_fs, 1}); auto storage_policy = std::make_shared(); storage_policy->name = "TabletCooldownTest"; @@ -149,6 +153,7 @@ TEST_F(TabletCooldownTest, normal) { // create tablet TCreateTabletReq request; create_tablet_request_with_sequence_col(10005, 270068377, &request); + request.__set_replica_id(10009); Status st = k_engine->create_tablet(request); ASSERT_EQ(Status::OK(), st); @@ -226,6 +231,10 @@ TEST_F(TabletCooldownTest, normal) { // test cooldown tablet->set_storage_policy_id(kStoragePolicyId); st = tablet->cooldown(); // rowset [0-1] + ASSERT_NE(Status::OK(), st); + tablet->update_cooldown_conf(1, 10009); + // cooldown for upload node + st = tablet->cooldown(); // rowset [0-1] ASSERT_EQ(Status::OK(), st); st = tablet->cooldown(); // rowset [2-2] ASSERT_EQ(Status::OK(), st); From 0bca8b47e367ee97bfdb8e5cdfa0c25b2dd1b072 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 15:05:40 +0800 Subject: [PATCH 02/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 50c1107aaae9b2..23de9c24adc29d 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -24,6 +24,7 @@ #include "exec/tablet_info.h" #include "gen_cpp/internal_service.pb.h" #include "io/fs/s3_file_system.h" +#include "io/fs/remote_file_system.h" #include "olap/delta_writer.h" #include "olap/rowset/beta_rowset.h" #include "olap/storage_engine.h" @@ -46,19 +47,12 @@ static constexpr int64_t kStoragePolicyId = 10002; // #define TabletCooldownTest DISABLED_TabletCooldownTest class TabletCooldownTest : public testing::Test { public: - class S3ClientMock : public Aws::S3::S3Client { - S3ClientMock() {} - - S3ClientMock(const Aws::Auth::AWSCredentials &credentials, - const Aws::Client::ClientConfiguration &clientConfiguration, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy signPayloads, - bool use_virtual_addressing) - : Aws::S3::S3Client(credentials, clientConfiguration, signPayloads, - use_virtual_addressing) {} + class S3FileSystemMock : public io::RemoteFileSystem { + S3FileSystemMock() {} }; static void SetUpTestSuite() { - S3ClientMock s3_fs; + S3FileSystemMock s3_fs; put_storage_resource(kResourceId, {s3_fs, 1}); auto storage_policy = std::make_shared(); storage_policy->name = "TabletCooldownTest"; From dd5f761c23ac76b772c2c9bae16770156f2e244c Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 16:46:55 +0800 Subject: [PATCH 03/36] add ut for cooldown on be --- be/test/CMakeLists.txt | 1 + be/test/io/fs/remote_file_system_mock.cpp | 71 +++++++++++++++++++++++ be/test/io/fs/remote_file_system_mock.h | 59 +++++++++++++++++++ be/test/olap/tablet_cooldown_test.cpp | 10 ++-- 4 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 be/test/io/fs/remote_file_system_mock.cpp create mode 100644 be/test/io/fs/remote_file_system_mock.h diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index dd6d542b0f675a..6b06fc375524ed 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -71,6 +71,7 @@ set(HTTP_TEST_FILES set(IO_TEST_FILES io/cache/remote_file_cache_test.cpp io/cache/file_block_cache_test.cpp + io/fs/remote_file_system_mock.cpp ) set(OLAP_TEST_FILES olap/engine_storage_migration_task_test.cpp diff --git a/be/test/io/fs/remote_file_system_mock.cpp b/be/test/io/fs/remote_file_system_mock.cpp new file mode 100644 index 00000000000000..b0f54a482e72f2 --- /dev/null +++ b/be/test/io/fs/remote_file_system_mock.cpp @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "remote_file_system_mock.h" + +Status S3FileSystemMock::create_file(const Path& path, FileWriterPtr* writer) { + return Status::OK(); +} + +Status S3FileSystemMock::open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) { + return Status::OK(); +} + +Status S3FileSystemMock::delete_file(const Path& path) { + return Status::OK(); +} + +Status S3FileSystemMock::create_directory(const Path& path) { + return Status::OK(); +} + +Status S3FileSystemMock::delete_directory(const Path& path) { + return Status::OK(); +} + +Status S3FileSystemMock::link_file(const Path& src, const Path& dest) { + return Status::OK(); +} + +Status S3FileSystemMock::exists(const Path& path, bool* res) { + return Status::OK(); +} + +Status S3FileSystemMock::file_size(const Path& path, size_t* file_size) { + return Status::OK(); +} + +Status S3FileSystemMock::list(const Path& path, std::vector* files) { + return Status::OK(); +} + +Status S3FileSystemMock::upload(const Path& local_path, const Path& dest_path) { + return Status::OK(); +} + +Status S3FileSystemMock::batch_upload(const std::vector& local_paths, + const std::vector& dest_paths) { + return Status::OK(); +} + +Status S3FileSystemMock::batch_delete(const std::vector& paths) { + return Status::OK(); +} + +Status S3FileSystemMock::connect() { + return Status::OK(); +} diff --git a/be/test/io/fs/remote_file_system_mock.h b/be/test/io/fs/remote_file_system_mock.h new file mode 100644 index 00000000000000..fd6c22688194c0 --- /dev/null +++ b/be/test/io/fs/remote_file_system_mock.h @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "io/fs/remote_file_system.h" + +namespace doris { +namespace io { + +public class S3FileSystemMock : public io::RemoteFileSystem { + S3FileSystemMock(Path&& root_path, std::string&& id, FileSystemType type) + : RemoteFileSystem(std::move(root_path), std::move(id), type) {} + ~S3FileSystem() override; + + Status create_file(const Path& path, FileWriterPtr* writer) override; + + Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override; + + Status delete_file(const Path& path) override; + + Status create_directory(const Path& path) override; + + Status delete_directory(const Path& path) override; + + Status link_file(const Path& src, const Path& dest) override; + + Status exists(const Path& path, bool* res) const override; + + Status file_size(const Path& path, size_t* file_size) const override; + + Status list(const Path& path, std::vector* files) override; + + Status upload(const Path& local_path, const Path& dest_path) override; + + Status batch_upload(const std::vector& local_paths, + const std::vector& dest_paths) override; + + Status batch_delete(const std::vector& paths) override; + + Status connect() override; +}; + +} // namespace io +} // namespace doris diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 23de9c24adc29d..31d62e04265e95 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -47,13 +47,11 @@ static constexpr int64_t kStoragePolicyId = 10002; // #define TabletCooldownTest DISABLED_TabletCooldownTest class TabletCooldownTest : public testing::Test { public: - class S3FileSystemMock : public io::RemoteFileSystem { - S3FileSystemMock() {} - }; - static void SetUpTestSuite() { - S3FileSystemMock s3_fs; - put_storage_resource(kResourceId, {s3_fs, 1}); + io::FileSystemSPtr s3_fs(new io::S3FileSystemMock("test_path", "id", + io::FileSystemType::S3)); + StorageResource resource = {s3_fs, 1}; + put_storage_resource(kResourceId, resource); auto storage_policy = std::make_shared(); storage_policy->name = "TabletCooldownTest"; storage_policy->version = 1; From 9070c4c5744a12a1fee802329218193834e23812 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 16:58:05 +0800 Subject: [PATCH 04/36] add ut for cooldown on be --- be/test/CMakeLists.txt | 1 - be/test/io/fs/remote_file_system_mock.cpp | 71 ----------------------- be/test/io/fs/remote_file_system_mock.h | 59 ------------------- be/test/olap/tablet_cooldown_test.cpp | 64 +++++++++++++++++++- 4 files changed, 62 insertions(+), 133 deletions(-) delete mode 100644 be/test/io/fs/remote_file_system_mock.cpp delete mode 100644 be/test/io/fs/remote_file_system_mock.h diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt index 6b06fc375524ed..dd6d542b0f675a 100644 --- a/be/test/CMakeLists.txt +++ b/be/test/CMakeLists.txt @@ -71,7 +71,6 @@ set(HTTP_TEST_FILES set(IO_TEST_FILES io/cache/remote_file_cache_test.cpp io/cache/file_block_cache_test.cpp - io/fs/remote_file_system_mock.cpp ) set(OLAP_TEST_FILES olap/engine_storage_migration_task_test.cpp diff --git a/be/test/io/fs/remote_file_system_mock.cpp b/be/test/io/fs/remote_file_system_mock.cpp deleted file mode 100644 index b0f54a482e72f2..00000000000000 --- a/be/test/io/fs/remote_file_system_mock.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "remote_file_system_mock.h" - -Status S3FileSystemMock::create_file(const Path& path, FileWriterPtr* writer) { - return Status::OK(); -} - -Status S3FileSystemMock::open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) { - return Status::OK(); -} - -Status S3FileSystemMock::delete_file(const Path& path) { - return Status::OK(); -} - -Status S3FileSystemMock::create_directory(const Path& path) { - return Status::OK(); -} - -Status S3FileSystemMock::delete_directory(const Path& path) { - return Status::OK(); -} - -Status S3FileSystemMock::link_file(const Path& src, const Path& dest) { - return Status::OK(); -} - -Status S3FileSystemMock::exists(const Path& path, bool* res) { - return Status::OK(); -} - -Status S3FileSystemMock::file_size(const Path& path, size_t* file_size) { - return Status::OK(); -} - -Status S3FileSystemMock::list(const Path& path, std::vector* files) { - return Status::OK(); -} - -Status S3FileSystemMock::upload(const Path& local_path, const Path& dest_path) { - return Status::OK(); -} - -Status S3FileSystemMock::batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) { - return Status::OK(); -} - -Status S3FileSystemMock::batch_delete(const std::vector& paths) { - return Status::OK(); -} - -Status S3FileSystemMock::connect() { - return Status::OK(); -} diff --git a/be/test/io/fs/remote_file_system_mock.h b/be/test/io/fs/remote_file_system_mock.h deleted file mode 100644 index fd6c22688194c0..00000000000000 --- a/be/test/io/fs/remote_file_system_mock.h +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "io/fs/remote_file_system.h" - -namespace doris { -namespace io { - -public class S3FileSystemMock : public io::RemoteFileSystem { - S3FileSystemMock(Path&& root_path, std::string&& id, FileSystemType type) - : RemoteFileSystem(std::move(root_path), std::move(id), type) {} - ~S3FileSystem() override; - - Status create_file(const Path& path, FileWriterPtr* writer) override; - - Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override; - - Status delete_file(const Path& path) override; - - Status create_directory(const Path& path) override; - - Status delete_directory(const Path& path) override; - - Status link_file(const Path& src, const Path& dest) override; - - Status exists(const Path& path, bool* res) const override; - - Status file_size(const Path& path, size_t* file_size) const override; - - Status list(const Path& path, std::vector* files) override; - - Status upload(const Path& local_path, const Path& dest_path) override; - - Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) override; - - Status batch_delete(const std::vector& paths) override; - - Status connect() override; -}; - -} // namespace io -} // namespace doris diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 31d62e04265e95..9e74bbb4c12eb2 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -25,6 +25,7 @@ #include "gen_cpp/internal_service.pb.h" #include "io/fs/s3_file_system.h" #include "io/fs/remote_file_system.h" +#include "io/fs/remote_file_system_mock.h" #include "olap/delta_writer.h" #include "olap/rowset/beta_rowset.h" #include "olap/storage_engine.h" @@ -42,14 +43,73 @@ static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; +class RemoteFileSystemMock : public io::RemoteFileSystem { + RemoteFileSystemMock(Path&& root_path, std::string&& id, FileSystemType type) + : RemoteFileSystem(std::move(root_path), std::move(id), type) {} + ~RemoteFileSystemMock() override {} + + Status create_file(const Path& path, FileWriterPtr* writer) override { + return Status::OK(); + } + + Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override { + return Status::OK(); + } + + Status delete_file(const Path& path) override { + return Status::OK(); + } + + Status create_directory(const Path& path) override { + return Status::OK(); + } + + Status delete_directory(const Path& path) override { + return Status::OK(); + } + + Status link_file(const Path& src, const Path& dest) override { + return Status::OK(); + } + + Status exists(const Path& path, bool* res) const override { + return Status::OK(); + } + + Status file_size(const Path& path, size_t* file_size) const override { + return Status::OK(); + } + + Status list(const Path& path, std::vector* files) override { + return Status::OK(); + } + + Status upload(const Path& local_path, const Path& dest_path) override { + return Status::OK(); + } + + Status batch_upload(const std::vector& local_paths, + const std::vector& dest_paths) override { + return Status::OK(); + } + + Status batch_delete(const std::vector& paths) override { + return Status::OK(); + } + + Status connect() override { + return Status::OK(); + } +}; + // remove DISABLED_ when need run this test // #define TabletCooldownTest DISABLED_TabletCooldownTest class TabletCooldownTest : public testing::Test { public: static void SetUpTestSuite() { - io::FileSystemSPtr s3_fs(new io::S3FileSystemMock("test_path", "id", - io::FileSystemType::S3)); + io::FileSystemSPtr s3_fs(new RemoteFileSystemMock("test_path", "id", + io::FileSystemType::S3)); StorageResource resource = {s3_fs, 1}; put_storage_resource(kResourceId, resource); auto storage_policy = std::make_shared(); From bae5d9c0a77c1f275d251277f9b23967e94fd440 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 18:50:34 +0800 Subject: [PATCH 05/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 121 +++++++++++++++----------- 1 file changed, 71 insertions(+), 50 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 9e74bbb4c12eb2..f144f1ee10cd99 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -25,7 +25,6 @@ #include "gen_cpp/internal_service.pb.h" #include "io/fs/s3_file_system.h" #include "io/fs/remote_file_system.h" -#include "io/fs/remote_file_system_mock.h" #include "olap/delta_writer.h" #include "olap/rowset/beta_rowset.h" #include "olap/storage_engine.h" @@ -43,73 +42,94 @@ static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; -class RemoteFileSystemMock : public io::RemoteFileSystem { - RemoteFileSystemMock(Path&& root_path, std::string&& id, FileSystemType type) - : RemoteFileSystem(std::move(root_path), std::move(id), type) {} - ~RemoteFileSystemMock() override {} +using io::Path; - Status create_file(const Path& path, FileWriterPtr* writer) override { - return Status::OK(); - } +// remove DISABLED_ when need run this test +// #define TabletCooldownTest DISABLED_TabletCooldownTest +class TabletCooldownTest : public testing::Test { +class FileWriterMock : public io::FileWriter { + public: + FileWriterMock(Path path) : FileWriter(std::move(path)) {} - Status open_file(const Path& path, FileReaderSPtr* reader, IOContext* io_ctx) override { - return Status::OK(); - } + ~FileWriterMock() {} - Status delete_file(const Path& path) override { - return Status::OK(); - } + Status open() override { + return Status::OK(); + } - Status create_directory(const Path& path) override { - return Status::OK(); - } + Status write(const uint8_t *buf, size_t buf_len, size_t *written_len) override { + return Status::OK(); + } - Status delete_directory(const Path& path) override { - return Status::OK(); + Status close() override { + return Status::OK(); + } } - Status link_file(const Path& src, const Path& dest) override { - return Status::OK(); - } + class RemoteFileSystemMock : public io::RemoteFileSystem { + RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) + : RemoteFileSystem(std::move(root_path), std::move(id), type) {} + ~RemoteFileSystemMock() override {} - Status exists(const Path& path, bool* res) const override { - return Status::OK(); - } + Status create_file(const Path& path, io::FileWriterPtr* writer) override { + writer->reset(new FileWriterMock()); + return Status::OK(); + } - Status file_size(const Path& path, size_t* file_size) const override { - return Status::OK(); - } + Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { + return Status::OK(); + } - Status list(const Path& path, std::vector* files) override { - return Status::OK(); - } + Status delete_file(const Path& path) override { + return Status::OK(); + } - Status upload(const Path& local_path, const Path& dest_path) override { - return Status::OK(); - } + Status create_directory(const Path& path) override { + return Status::OK(); + } - Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) override { - return Status::OK(); - } + Status delete_directory(const Path& path) override { + return Status::OK(); + } - Status batch_delete(const std::vector& paths) override { - return Status::OK(); - } + Status link_file(const Path& src, const Path& dest) override { + return Status::OK(); + } - Status connect() override { - return Status::OK(); - } -}; + Status exists(const Path& path, bool* res) const override { + return Status::OK(); + } + Status file_size(const Path& path, size_t* file_size) const override { + return Status::OK(); + } + + Status list(const Path& path, std::vector* files) override { + return Status::OK(); + } + + Status upload(const Path& local_path, const Path& dest_path) override { + return Status::OK(); + } + + Status batch_upload(const std::vector& local_paths, + const std::vector& dest_paths) override { + return Status::OK(); + } + + Status batch_delete(const std::vector& paths) override { + return Status::OK(); + } + + Status connect() override { + return Status::OK(); + } + }; -// remove DISABLED_ when need run this test -// #define TabletCooldownTest DISABLED_TabletCooldownTest -class TabletCooldownTest : public testing::Test { public: static void SetUpTestSuite() { - io::FileSystemSPtr s3_fs(new RemoteFileSystemMock("test_path", "id", - io::FileSystemType::S3)); + io::FileSystemSPtr s3_fs(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), + io::FileSystemType::S3)); StorageResource resource = {s3_fs, 1}; put_storage_resource(kResourceId, resource); auto storage_policy = std::make_shared(); @@ -141,6 +161,7 @@ class TabletCooldownTest : public testing::Test { k_engine = nullptr; } } + }; static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash, From a2bfed7a5ed5b8844ae0516ee9265b8361bfdd39 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 19:10:13 +0800 Subject: [PATCH 06/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 186 ++++++++++++++------------ 1 file changed, 104 insertions(+), 82 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index f144f1ee10cd99..f70e83a4b96cad 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -23,6 +23,7 @@ #include "common/status.h" #include "exec/tablet_info.h" #include "gen_cpp/internal_service.pb.h" +#include "io/file_writer.h" #include "io/fs/s3_file_system.h" #include "io/fs/remote_file_system.h" #include "olap/delta_writer.h" @@ -42,95 +43,21 @@ static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; +class RemoteFileSystemMock; +class FileWriterMock; + +static io::FileSystemSPtr s_fs(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), + io::FileSystemType::S3)); +static io::FileWriterPtr s_writer(new FileWriterMock("test_path")); + using io::Path; // remove DISABLED_ when need run this test // #define TabletCooldownTest DISABLED_TabletCooldownTest class TabletCooldownTest : public testing::Test { -class FileWriterMock : public io::FileWriter { - public: - FileWriterMock(Path path) : FileWriter(std::move(path)) {} - - ~FileWriterMock() {} - - Status open() override { - return Status::OK(); - } - - Status write(const uint8_t *buf, size_t buf_len, size_t *written_len) override { - return Status::OK(); - } - - Status close() override { - return Status::OK(); - } - } - - class RemoteFileSystemMock : public io::RemoteFileSystem { - RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) - : RemoteFileSystem(std::move(root_path), std::move(id), type) {} - ~RemoteFileSystemMock() override {} - - Status create_file(const Path& path, io::FileWriterPtr* writer) override { - writer->reset(new FileWriterMock()); - return Status::OK(); - } - - Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return Status::OK(); - } - - Status delete_file(const Path& path) override { - return Status::OK(); - } - - Status create_directory(const Path& path) override { - return Status::OK(); - } - - Status delete_directory(const Path& path) override { - return Status::OK(); - } - - Status link_file(const Path& src, const Path& dest) override { - return Status::OK(); - } - - Status exists(const Path& path, bool* res) const override { - return Status::OK(); - } - - Status file_size(const Path& path, size_t* file_size) const override { - return Status::OK(); - } - - Status list(const Path& path, std::vector* files) override { - return Status::OK(); - } - - Status upload(const Path& local_path, const Path& dest_path) override { - return Status::OK(); - } - - Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) override { - return Status::OK(); - } - - Status batch_delete(const std::vector& paths) override { - return Status::OK(); - } - - Status connect() override { - return Status::OK(); - } - }; - public: static void SetUpTestSuite() { - io::FileSystemSPtr s3_fs(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), - io::FileSystemType::S3)); - StorageResource resource = {s3_fs, 1}; + StorageResource resource = {s_fs, 1}; put_storage_resource(kResourceId, resource); auto storage_policy = std::make_shared(); storage_policy->name = "TabletCooldownTest"; @@ -164,6 +91,101 @@ class FileWriterMock : public io::FileWriter { }; +class FileWriterMock : public io::FileWriter { +public: + FileWriterMock(Path path) : io::FileWriter(std::move(path)) {} + + ~FileWriterMock() {} + + Status close() override { + return Status::OK(); + } + + Status abort() override { + return Status::OK(); + } + + Status append(const Slice& data) override { + return Status::OK(); + } + + Status appendv(const Slice* data, size_t data_cnt) override { + return Status::OK(); + } + + Status write_at(size_t offset, const Slice& data) override { + return Status::OK(); + } + + Status finalize() override { + return Status::OK(); + } + + size_t bytes_appended() const override { return 0; } + + FileSystemSPtr fs() const override { return fs; } +}; + +class RemoteFileSystemMock : public io::RemoteFileSystem { + RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) + : RemoteFileSystem(std::move(root_path), std::move(id), type) {} + ~RemoteFileSystemMock() override {} + + Status create_file(const Path& path, io::FileWriterPtr* writer) override { + *writer = s_writer; + return Status::OK(); + } + + Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { + return Status::OK(); + } + + Status delete_file(const Path& path) override { + return Status::OK(); + } + + Status create_directory(const Path& path) override { + return Status::OK(); + } + + Status delete_directory(const Path& path) override { + return Status::OK(); + } + + Status link_file(const Path& src, const Path& dest) override { + return Status::OK(); + } + + Status exists(const Path& path, bool* res) const override { + return Status::OK(); + } + + Status file_size(const Path& path, size_t* file_size) const override { + return Status::OK(); + } + + Status list(const Path& path, std::vector* files) override { + return Status::OK(); + } + + Status upload(const Path& local_path, const Path& dest_path) override { + return Status::OK(); + } + + Status batch_upload(const std::vector& local_paths, + const std::vector& dest_paths) override { + return Status::OK(); + } + + Status batch_delete(const std::vector& paths) override { + return Status::OK(); + } + + Status connect() override { + return Status::OK(); + } +}; + static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash, TCreateTabletReq* request) { request->tablet_id = tablet_id; From 8153afecc3e52f55f1e5977bdbc487aecf2df939 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 19:34:56 +0800 Subject: [PATCH 07/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 95 +++++++++++++-------------- 1 file changed, 46 insertions(+), 49 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index f70e83a4b96cad..89b162db0087de 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -43,54 +43,6 @@ static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; -class RemoteFileSystemMock; -class FileWriterMock; - -static io::FileSystemSPtr s_fs(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), - io::FileSystemType::S3)); -static io::FileWriterPtr s_writer(new FileWriterMock("test_path")); - -using io::Path; - -// remove DISABLED_ when need run this test -// #define TabletCooldownTest DISABLED_TabletCooldownTest -class TabletCooldownTest : public testing::Test { -public: - static void SetUpTestSuite() { - StorageResource resource = {s_fs, 1}; - put_storage_resource(kResourceId, resource); - auto storage_policy = std::make_shared(); - storage_policy->name = "TabletCooldownTest"; - storage_policy->version = 1; - storage_policy->resource_id = kResourceId; - put_storage_policy(kStoragePolicyId, storage_policy); - - constexpr uint32_t MAX_PATH_LEN = 1024; - char buffer[MAX_PATH_LEN]; - EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); - config::storage_root_path = std::string(buffer) + "/" + kTestDir; - config::min_file_descriptor_number = 1000; - - FileUtils::remove_all(config::storage_root_path); - FileUtils::create_dir(config::storage_root_path); - - std::vector paths {{config::storage_root_path, -1}}; - - EngineOptions options; - options.store_paths = paths; - doris::StorageEngine::open(options, &k_engine); - } - - static void TearDownTestSuite() { - if (k_engine != nullptr) { - k_engine->stop(); - delete k_engine; - k_engine = nullptr; - } - } - -}; - class FileWriterMock : public io::FileWriter { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) {} @@ -123,7 +75,7 @@ class FileWriterMock : public io::FileWriter { size_t bytes_appended() const override { return 0; } - FileSystemSPtr fs() const override { return fs; } + io::FileSystemSPtr fs() const override { return fs; } }; class RemoteFileSystemMock : public io::RemoteFileSystem { @@ -186,6 +138,51 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { } }; +static io::FileSystemSPtr s_fs(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), + io::FileSystemType::S3)); +static io::FileWriterPtr s_writer(new FileWriterMock("test_path")); + +using io::Path; + +// remove DISABLED_ when need run this test +// #define TabletCooldownTest DISABLED_TabletCooldownTest +class TabletCooldownTest : public testing::Test { +public: + static void SetUpTestSuite() { + StorageResource resource = {s_fs, 1}; + put_storage_resource(kResourceId, resource); + auto storage_policy = std::make_shared(); + storage_policy->name = "TabletCooldownTest"; + storage_policy->version = 1; + storage_policy->resource_id = kResourceId; + put_storage_policy(kStoragePolicyId, storage_policy); + + constexpr uint32_t MAX_PATH_LEN = 1024; + char buffer[MAX_PATH_LEN]; + EXPECT_NE(getcwd(buffer, MAX_PATH_LEN), nullptr); + config::storage_root_path = std::string(buffer) + "/" + kTestDir; + config::min_file_descriptor_number = 1000; + + FileUtils::remove_all(config::storage_root_path); + FileUtils::create_dir(config::storage_root_path); + + std::vector paths {{config::storage_root_path, -1}}; + + EngineOptions options; + options.store_paths = paths; + doris::StorageEngine::open(options, &k_engine); + } + + static void TearDownTestSuite() { + if (k_engine != nullptr) { + k_engine->stop(); + delete k_engine; + k_engine = nullptr; + } + } + +}; + static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash, TCreateTabletReq* request) { request->tablet_id = tablet_id; From 056da2bdf13203449e8db701e733aacd1e816834 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 19:44:05 +0800 Subject: [PATCH 08/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 89b162db0087de..e68596e54ace8e 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -43,6 +43,11 @@ static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; +using io::Path; + +static io::FileSystemSPtr s_fs; +static io::FileWriterPtr s_writer; + class FileWriterMock : public io::FileWriter { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) {} @@ -75,7 +80,7 @@ class FileWriterMock : public io::FileWriter { size_t bytes_appended() const override { return 0; } - io::FileSystemSPtr fs() const override { return fs; } + io::FileSystemSPtr fs() const override { return s_fs; } }; class RemoteFileSystemMock : public io::RemoteFileSystem { @@ -138,17 +143,12 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { } }; -static io::FileSystemSPtr s_fs(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), - io::FileSystemType::S3)); -static io::FileWriterPtr s_writer(new FileWriterMock("test_path")); - -using io::Path; - -// remove DISABLED_ when need run this test -// #define TabletCooldownTest DISABLED_TabletCooldownTest class TabletCooldownTest : public testing::Test { public: static void SetUpTestSuite() { + s_fs.reset(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), + io::FileSystemType::S3)); + s_writer.reset(new FileWriterMock("test_path")); StorageResource resource = {s_fs, 1}; put_storage_resource(kResourceId, resource); auto storage_policy = std::make_shared(); From 95ad9dbb163d57a2d39df2ddc96c51de05794752 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Wed, 1 Mar 2023 19:50:39 +0800 Subject: [PATCH 09/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index e68596e54ace8e..f9de855bffef0b 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -46,7 +46,6 @@ static constexpr int64_t kStoragePolicyId = 10002; using io::Path; static io::FileSystemSPtr s_fs; -static io::FileWriterPtr s_writer; class FileWriterMock : public io::FileWriter { public: @@ -89,7 +88,7 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { ~RemoteFileSystemMock() override {} Status create_file(const Path& path, io::FileWriterPtr* writer) override { - *writer = s_writer; + *writer = std::make_unique("test_path"); return Status::OK(); } @@ -148,7 +147,6 @@ class TabletCooldownTest : public testing::Test { static void SetUpTestSuite() { s_fs.reset(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), io::FileSystemType::S3)); - s_writer.reset(new FileWriterMock("test_path")); StorageResource resource = {s_fs, 1}; put_storage_resource(kResourceId, resource); auto storage_policy = std::make_shared(); From 4567bd0cea3f5359189e33795231a06f8261d0d9 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 13:24:01 +0800 Subject: [PATCH 10/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 146 +++++++++++--------------- 1 file changed, 61 insertions(+), 85 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index f9de855bffef0b..265de38bfb7b89 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -23,6 +23,8 @@ #include "common/status.h" #include "exec/tablet_info.h" #include "gen_cpp/internal_service.pb.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" #include "io/file_writer.h" #include "io/fs/s3_file_system.h" #include "io/fs/remote_file_system.h" @@ -50,96 +52,40 @@ static io::FileSystemSPtr s_fs; class FileWriterMock : public io::FileWriter { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) {} - ~FileWriterMock() {} - Status close() override { - return Status::OK(); - } - - Status abort() override { - return Status::OK(); - } - - Status append(const Slice& data) override { - return Status::OK(); - } - - Status appendv(const Slice* data, size_t data_cnt) override { - return Status::OK(); - } - - Status write_at(size_t offset, const Slice& data) override { - return Status::OK(); - } - - Status finalize() override { - return Status::OK(); - } - - size_t bytes_appended() const override { return 0; } - - io::FileSystemSPtr fs() const override { return s_fs; } + MOCK_METHOD0(close, Status()); + MOCK_METHOD0(abort, Status()); + MOCK_METHOD1(append, Status(const Slice& data)); + MOCK_METHOD2(appendv, Status(const Slice* data, size_t data_cnt)); + MOCK_METHOD2(write_at, Status(size_t offset, const Slice& data)); + MOCK_METHOD0(finalize, Status()); + MOCK_METHOD0(bytes_appended, size_t()); + MOCK_METHOD0(fs, io::FileSystemSPtr()); }; class RemoteFileSystemMock : public io::RemoteFileSystem { RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) : RemoteFileSystem(std::move(root_path), std::move(id), type) {} - ~RemoteFileSystemMock() override {} + ~RemoteFileSystemMock() {} Status create_file(const Path& path, io::FileWriterPtr* writer) override { *writer = std::make_unique("test_path"); return Status::OK(); } - - Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return Status::OK(); - } - - Status delete_file(const Path& path) override { - return Status::OK(); - } - - Status create_directory(const Path& path) override { - return Status::OK(); - } - - Status delete_directory(const Path& path) override { - return Status::OK(); - } - - Status link_file(const Path& src, const Path& dest) override { - return Status::OK(); - } - - Status exists(const Path& path, bool* res) const override { - return Status::OK(); - } - - Status file_size(const Path& path, size_t* file_size) const override { - return Status::OK(); - } - - Status list(const Path& path, std::vector* files) override { - return Status::OK(); - } - - Status upload(const Path& local_path, const Path& dest_path) override { - return Status::OK(); - } - - Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) override { - return Status::OK(); - } - - Status batch_delete(const std::vector& paths) override { - return Status::OK(); - } - - Status connect() override { - return Status::OK(); - } + MOCK_METHOD3(open_file, Status(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx)); + MOCK_METHOD1(delete_file, Status(const Path& path)); + MOCK_METHOD1(create_directory, Status(const Path& path)); + MOCK_METHOD1(delete_directory, Status(const Path& path)); + MOCK_METHOD2(link_file, Status(const Path& src, const Path& dest)); + MOCK_METHOD2(exists, Status(const Path& path, bool* res)); + MOCK_METHOD2(file_size, Status(const Path& path, size_t* file_size)); + MOCK_METHOD2(list, Status(const Path& path, std::vector* files)); + MOCK_METHOD2(upload, Status(const Path& local_path, const Path& dest_path)); + MOCK_METHOD2(batch_upload, Status(const std::vector& local_paths, + const std::vector& dest_paths)); + MOCK_METHOD1(batch_delete, Status(const std::vector& paths)); + MOCK_METHOD0(connect, Status()); }; class TabletCooldownTest : public testing::Test { @@ -179,6 +125,9 @@ class TabletCooldownTest : public testing::Test { } } +private: + + static io::FileWriterPtr* _file_writer; }; static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash, @@ -240,6 +189,40 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() { } TEST_F(TabletCooldownTest, normal) { + EXPECT_CALL(_file_writer, close()).WillRepeatedly(::testing::Return(Status::OK())); + EXPECT_CALL(_file_writer, abort()).WillRepeatedly(::testing::Return(Status::OK())); + EXPECT_CALL(_file_writer, append(::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(_file_writer, appendv(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(_file_writer, write_at(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(_file_writer, finalize()).WillRepeatedly(::testing::Return(Status::OK())); + EXPECT_CALL(_file_writer, bytes_appended()).WillRepeatedly(::testing::Return(1)); + EXPECT_CALL(_file_writer, fs()).WillRepeatedly(::testing::Return(s_fs)) + + EXPECT_CALL(s_fs, open_file(::testing::_, ::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, delete_file(::testing::_)).WillRepeatedly(::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, create_directory(::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, delete_directory(::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, link_file(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, exists(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, file_size(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, list(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, upload(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, batch_upload(::testing::_, ::testing::_)).WillRepeatedly( + ::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, batch_delete(::testing::_)).WillRepeatedly(::testing::Return(Status::OK())); + EXPECT_CALL(s_fs, connect()).WillRepeatedly(::testing::Return(Status::OK())); + // create tablet TCreateTabletReq request; create_tablet_request_with_sequence_col(10005, 270068377, &request); @@ -330,13 +313,6 @@ TEST_F(TabletCooldownTest, normal) { ASSERT_EQ(Status::OK(), st); auto rs = tablet->get_rowset_by_version({2, 2}); ASSERT_FALSE(rs->is_local()); - - // test read - ASSERT_EQ(Status::OK(), st); - std::vector segments; - st = std::static_pointer_cast(rs)->load_segments(&segments); - ASSERT_EQ(Status::OK(), st); - ASSERT_EQ(segments.size(), 1); } } // namespace doris From af7103d51d6e18198058f4ae076c70007c3b3fe0 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 14:08:41 +0800 Subject: [PATCH 11/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 63 ++++++++++++--------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 265de38bfb7b89..490f6734cb1ac3 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -39,6 +39,11 @@ namespace doris { +using ::testing::_; +using ::testing::DoAll; +using ::testing::Return; +using ::testing::SetArgPointee; + static StorageEngine* k_engine = nullptr; static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; @@ -78,8 +83,8 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { MOCK_METHOD1(create_directory, Status(const Path& path)); MOCK_METHOD1(delete_directory, Status(const Path& path)); MOCK_METHOD2(link_file, Status(const Path& src, const Path& dest)); - MOCK_METHOD2(exists, Status(const Path& path, bool* res)); - MOCK_METHOD2(file_size, Status(const Path& path, size_t* file_size)); + MOCK_CONST_METHOD2(exists, Status(const Path& path, bool* res)); + MOCK_CONST_METHOD2(file_size, Status(const Path& path, size_t* file_size)); MOCK_METHOD2(list, Status(const Path& path, std::vector* files)); MOCK_METHOD2(upload, Status(const Path& local_path, const Path& dest_path)); MOCK_METHOD2(batch_upload, Status(const std::vector& local_paths, @@ -189,39 +194,27 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() { } TEST_F(TabletCooldownTest, normal) { - EXPECT_CALL(_file_writer, close()).WillRepeatedly(::testing::Return(Status::OK())); - EXPECT_CALL(_file_writer, abort()).WillRepeatedly(::testing::Return(Status::OK())); - EXPECT_CALL(_file_writer, append(::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(_file_writer, appendv(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(_file_writer, write_at(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(_file_writer, finalize()).WillRepeatedly(::testing::Return(Status::OK())); - EXPECT_CALL(_file_writer, bytes_appended()).WillRepeatedly(::testing::Return(1)); - EXPECT_CALL(_file_writer, fs()).WillRepeatedly(::testing::Return(s_fs)) - - EXPECT_CALL(s_fs, open_file(::testing::_, ::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, delete_file(::testing::_)).WillRepeatedly(::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, create_directory(::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, delete_directory(::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, link_file(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, exists(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, file_size(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, list(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, upload(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, batch_upload(::testing::_, ::testing::_)).WillRepeatedly( - ::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, batch_delete(::testing::_)).WillRepeatedly(::testing::Return(Status::OK())); - EXPECT_CALL(s_fs, connect()).WillRepeatedly(::testing::Return(Status::OK())); + EXPECT_CALL(*_file_writer, close()).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*_file_writer, abort()).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*_file_writer, append(_)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*_file_writer, appendv(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*_file_writer, write_at(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*_file_writer, finalize()).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*_file_writer, bytes_appended()).WillRepeatedly(Return(1)); + EXPECT_CALL(*_file_writer, fs()).WillRepeatedly(Return(s_fs)) + + EXPECT_CALL(*s_fs, open_file(_, _, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, delete_file(_)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, create_directory(_)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, delete_directory(_)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, link_file(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, exists(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, file_size(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, list(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, upload(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, batch_upload(_, _)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, batch_delete(_)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*s_fs, connect()).WillRepeatedly(Return(Status::OK())); // create tablet TCreateTabletReq request; From 74eac861fafc2aab15ad41af6a60c395b0d86b51 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 16:36:48 +0800 Subject: [PATCH 12/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 129 ++++++++++++++++++-------- 1 file changed, 92 insertions(+), 37 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 490f6734cb1ac3..8692858425a438 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -54,46 +54,101 @@ using io::Path; static io::FileSystemSPtr s_fs; -class FileWriterMock : public io::FileWriter { -public: - FileWriterMock(Path path) : io::FileWriter(std::move(path)) {} - ~FileWriterMock() {} - - MOCK_METHOD0(close, Status()); - MOCK_METHOD0(abort, Status()); - MOCK_METHOD1(append, Status(const Slice& data)); - MOCK_METHOD2(appendv, Status(const Slice* data, size_t data_cnt)); - MOCK_METHOD2(write_at, Status(size_t offset, const Slice& data)); - MOCK_METHOD0(finalize, Status()); - MOCK_METHOD0(bytes_appended, size_t()); - MOCK_METHOD0(fs, io::FileSystemSPtr()); -}; +class TabletCooldownTest : public testing::Test { + class FileWriterMock : public io::FileWriter { + public: + FileWriterMock(Path path) : io::FileWriter(std::move(path)) {} -class RemoteFileSystemMock : public io::RemoteFileSystem { - RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) - : RemoteFileSystem(std::move(root_path), std::move(id), type) {} - ~RemoteFileSystemMock() {} + ~FileWriterMock() {} - Status create_file(const Path& path, io::FileWriterPtr* writer) override { - *writer = std::make_unique("test_path"); - return Status::OK(); - } - MOCK_METHOD3(open_file, Status(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx)); - MOCK_METHOD1(delete_file, Status(const Path& path)); - MOCK_METHOD1(create_directory, Status(const Path& path)); - MOCK_METHOD1(delete_directory, Status(const Path& path)); - MOCK_METHOD2(link_file, Status(const Path& src, const Path& dest)); - MOCK_CONST_METHOD2(exists, Status(const Path& path, bool* res)); - MOCK_CONST_METHOD2(file_size, Status(const Path& path, size_t* file_size)); - MOCK_METHOD2(list, Status(const Path& path, std::vector* files)); - MOCK_METHOD2(upload, Status(const Path& local_path, const Path& dest_path)); - MOCK_METHOD2(batch_upload, Status(const std::vector& local_paths, - const std::vector& dest_paths)); - MOCK_METHOD1(batch_delete, Status(const std::vector& paths)); - MOCK_METHOD0(connect, Status()); -}; + Status close() override { + return Status::OK(); + } -class TabletCooldownTest : public testing::Test { + Status abort() override { + return Status::OK(); + } + + Status append(const Slice& data) override { + return Status::OK(); + } + + Status appendv(const Slice* data, size_t data_cnt) override { + return Status::OK(); + } + + Status write_at(size_t offset, const Slice& data) override { + return Status::OK(); + } + + Status finalize() override { + return Status::OK(); + } + + size_t bytes_appended() const override { return 0; } + + io::FileSystemSPtr fs() const override { return s_fs; } + }; + + class RemoteFileSystemMock : public io::RemoteFileSystem { + RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) + : RemoteFileSystem(std::move(root_path), std::move(id), type) {} + ~RemoteFileSystemMock() override {} + + Status create_file(const Path& path, io::FileWriterPtr* writer) override { + *writer = std::make_unique("test_path"); + return Status::OK(); + } + + Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { + return Status::OK(); + } + + Status delete_file(const Path& path) override { + return Status::OK(); + } + + Status create_directory(const Path& path) override { + return Status::OK(); + } + + Status delete_directory(const Path& path) override { + return Status::OK(); + } + + Status link_file(const Path& src, const Path& dest) override { + return Status::OK(); + } + + Status exists(const Path& path, bool* res) const override { + return Status::OK(); + } + + Status file_size(const Path& path, size_t* file_size) const override { + return Status::OK(); + } + + Status list(const Path& path, std::vector* files) override { + return Status::OK(); + } + + Status upload(const Path& local_path, const Path& dest_path) override { + return Status::OK(); + } + + Status batch_upload(const std::vector& local_paths, + const std::vector& dest_paths) override { + return Status::OK(); + } + + Status batch_delete(const std::vector& paths) override { + return Status::OK(); + } + + Status connect() override { + return Status::OK(); + } + }; public: static void SetUpTestSuite() { s_fs.reset(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), From f295cf763aaffe2446bcc47e8b3826e5ee0d4913 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 16:37:50 +0800 Subject: [PATCH 13/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 8692858425a438..1a2ca093f9ca2d 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -249,28 +249,6 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() { } TEST_F(TabletCooldownTest, normal) { - EXPECT_CALL(*_file_writer, close()).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*_file_writer, abort()).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*_file_writer, append(_)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*_file_writer, appendv(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*_file_writer, write_at(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*_file_writer, finalize()).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*_file_writer, bytes_appended()).WillRepeatedly(Return(1)); - EXPECT_CALL(*_file_writer, fs()).WillRepeatedly(Return(s_fs)) - - EXPECT_CALL(*s_fs, open_file(_, _, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, delete_file(_)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, create_directory(_)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, delete_directory(_)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, link_file(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, exists(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, file_size(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, list(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, upload(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, batch_upload(_, _)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, batch_delete(_)).WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(*s_fs, connect()).WillRepeatedly(Return(Status::OK())); - // create tablet TCreateTabletReq request; create_tablet_request_with_sequence_col(10005, 270068377, &request); From 4212a54df9c450b57b6416da8fd6cb5e5d00cbbd Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 17:20:29 +0800 Subject: [PATCH 14/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 1a2ca093f9ca2d..99b6841310caa5 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -23,8 +23,6 @@ #include "common/status.h" #include "exec/tablet_info.h" #include "gen_cpp/internal_service.pb.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" #include "io/file_writer.h" #include "io/fs/s3_file_system.h" #include "io/fs/remote_file_system.h" @@ -49,6 +47,9 @@ static StorageEngine* k_engine = nullptr; static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; +static constexpr int64_t kTabletId = 10005; +static constexpr int64_t kReplicaId = 10009; +static constexpr int64_t kSchemaHash = 270068377; using io::Path; @@ -251,8 +252,8 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() { TEST_F(TabletCooldownTest, normal) { // create tablet TCreateTabletReq request; - create_tablet_request_with_sequence_col(10005, 270068377, &request); - request.__set_replica_id(10009); + create_tablet_request_with_sequence_col(kTabletId, kSchemaHash, &request); + request.__set_replica_id(kReplicaId); Status st = k_engine->create_tablet(request); ASSERT_EQ(Status::OK(), st); @@ -267,7 +268,7 @@ TEST_F(TabletCooldownTest, normal) { PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); - WriteRequest write_req = {10005, 270068377, WriteType::LOAD, 20003, 30003, + WriteRequest write_req = {kTabletId, kSchemaHash, WriteType::LOAD, 20003, 30003, load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); @@ -331,7 +332,7 @@ TEST_F(TabletCooldownTest, normal) { tablet->set_storage_policy_id(kStoragePolicyId); st = tablet->cooldown(); // rowset [0-1] ASSERT_NE(Status::OK(), st); - tablet->update_cooldown_conf(1, 10009); + tablet->update_cooldown_conf(1, kReplicaId); // cooldown for upload node st = tablet->cooldown(); // rowset [0-1] ASSERT_EQ(Status::OK(), st); From cb4dd81d5731fb0238b1bee84a71cab5c54732ba Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 17:32:05 +0800 Subject: [PATCH 15/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 99b6841310caa5..fc0a61e082dbfe 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -37,11 +37,6 @@ namespace doris { -using ::testing::_; -using ::testing::DoAll; -using ::testing::Return; -using ::testing::SetArgPointee; - static StorageEngine* k_engine = nullptr; static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; From a5d074e0c27bec347944164593a5b6ba21d6c125 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 18:17:17 +0800 Subject: [PATCH 16/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 64 ++++++++++++++++++--------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index fc0a61e082dbfe..aee68ff20e3cf1 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -53,98 +53,122 @@ static io::FileSystemSPtr s_fs; class TabletCooldownTest : public testing::Test { class FileWriterMock : public io::FileWriter { public: - FileWriterMock(Path path) : io::FileWriter(std::move(path)) {} + FileWriterMock(Path path) : io::FileWriter(path) { + *_local_file_writer = std::make_unique(fmt::format( + "{}/{}", kTestDir, path.str())); + } ~FileWriterMock() {} Status close() override { - return Status::OK(); + return _local_file_writer->close(); } Status abort() override { - return Status::OK(); + return _local_file_writer->abort(); } Status append(const Slice& data) override { - return Status::OK(); + return _local_file_writer->append(data); } Status appendv(const Slice* data, size_t data_cnt) override { - return Status::OK(); + return _local_file_writer->appendv(data, data_cnt); } Status write_at(size_t offset, const Slice& data) override { - return Status::OK(); + return _local_file_writer->write_at(offset, data); } Status finalize() override { - return Status::OK(); + return _local_file_writer->finalize(); } - size_t bytes_appended() const override { return 0; } + size_t bytes_appended() const override { return _local_file_writer->bytes_appended(); } io::FileSystemSPtr fs() const override { return s_fs; } + private: + std::unique_ptr _local_file_writer; }; class RemoteFileSystemMock : public io::RemoteFileSystem { RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) - : RemoteFileSystem(std::move(root_path), std::move(id), type) {} + : RemoteFileSystem(root_path, id, type) { + _local_fs = io::LocalFileSystem::create(fmt::format("{}/{}", kTestDir, + root_path.str())); + } ~RemoteFileSystemMock() override {} Status create_file(const Path& path, io::FileWriterPtr* writer) override { - *writer = std::make_unique("test_path"); + *writer = std::make_unique(path); return Status::OK(); } Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return Status::OK(); + return _local_fs->open_file(fmt::format("{}/{}", kTestDir, path.str()), reader, io_ctx); } Status delete_file(const Path& path) override { - return Status::OK(); + return _local_fs->delete_file(fmt::format("{}/{}", kTestDir, path.str())); } Status create_directory(const Path& path) override { - return Status::OK(); + return _local_fs->create_directory(fmt::format("{}/{}", kTestDir, path.str())); } Status delete_directory(const Path& path) override { - return Status::OK(); + return _local_fs->delete_directory(fmt::format("{}/{}", kTestDir, path.str())); } Status link_file(const Path& src, const Path& dest) override { - return Status::OK(); + return _local_fs->link_file(fmt::format("{}/{}", kTestDir, src.str()), + fmt::format("{}/{}", kTestDir, dest.str())); } Status exists(const Path& path, bool* res) const override { - return Status::OK(); + return _local_fs->exists(fmt::format("{}/{}", kTestDir, path.str()), res); } Status file_size(const Path& path, size_t* file_size) const override { - return Status::OK(); + return _local_fs->file_size(fmt::format("{}/{}", kTestDir, path.str()), file_size); } Status list(const Path& path, std::vector* files) override { + std::vector local_paths; + RETURN_IF_ERROR(_local_fs->list(fmt::format("{}/{}", kTestDir, path.str()), + &local_paths)); + for (Path path : local_paths) { + files->emplace_back(path.str().substr(kTestDir.size() + 1)); + } return Status::OK(); } Status upload(const Path& local_path, const Path& dest_path) override { - return Status::OK(); + return link_file(local_path, dest_path); } Status batch_upload(const std::vector& local_paths, const std::vector& dest_paths) override { + for (int i = 0; i < local_paths.size(); ++i) { + RETURN_IF_ERROR(link_file(local_paths[i], dest_paths[i])); + } return Status::OK(); } Status batch_delete(const std::vector& paths) override { + for (int i = 0; i < paths.size(); ++i) { + RETURN_IF_ERROR(delete_file(paths[i])); + } return Status::OK(); } Status connect() override { return Status::OK(); } + private: + std::shared_ptr _local_fs; }; + public: static void SetUpTestSuite() { s_fs.reset(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), @@ -180,10 +204,6 @@ class TabletCooldownTest : public testing::Test { k_engine = nullptr; } } - -private: - - static io::FileWriterPtr* _file_writer; }; static void create_tablet_request_with_sequence_col(int64_t tablet_id, int32_t schema_hash, From c88d836eff6f01bfac3d750fe9147895d1be816a Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 18:27:19 +0800 Subject: [PATCH 17/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index aee68ff20e3cf1..78a7b0ef8ee903 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -53,9 +53,9 @@ static io::FileSystemSPtr s_fs; class TabletCooldownTest : public testing::Test { class FileWriterMock : public io::FileWriter { public: - FileWriterMock(Path path) : io::FileWriter(path) { - *_local_file_writer = std::make_unique(fmt::format( - "{}/{}", kTestDir, path.str())); + FileWriterMock(Path path) : io::FileWriter(std::move(path)) { + _local_file_writer = std::make_unique(fmt::format( + "{}/{}", kTestDir, _path.string())); } ~FileWriterMock() {} @@ -88,14 +88,14 @@ class TabletCooldownTest : public testing::Test { io::FileSystemSPtr fs() const override { return s_fs; } private: - std::unique_ptr _local_file_writer; + std::unique_ptr _local_file_writer; }; class RemoteFileSystemMock : public io::RemoteFileSystem { RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) - : RemoteFileSystem(root_path, id, type) { + : RemoteFileSystem(std::move(root_path), std::move(id), type) { _local_fs = io::LocalFileSystem::create(fmt::format("{}/{}", kTestDir, - root_path.str())); + _root_path.string())); } ~RemoteFileSystemMock() override {} @@ -105,40 +105,40 @@ class TabletCooldownTest : public testing::Test { } Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return _local_fs->open_file(fmt::format("{}/{}", kTestDir, path.str()), reader, io_ctx); + return _local_fs->open_file(fmt::format("{}/{}", kTestDir, path.string()), reader, io_ctx); } Status delete_file(const Path& path) override { - return _local_fs->delete_file(fmt::format("{}/{}", kTestDir, path.str())); + return _local_fs->delete_file(fmt::format("{}/{}", kTestDir, path.string())); } Status create_directory(const Path& path) override { - return _local_fs->create_directory(fmt::format("{}/{}", kTestDir, path.str())); + return _local_fs->create_directory(fmt::format("{}/{}", kTestDir, path.string())); } Status delete_directory(const Path& path) override { - return _local_fs->delete_directory(fmt::format("{}/{}", kTestDir, path.str())); + return _local_fs->delete_directory(fmt::format("{}/{}", kTestDir, path.string())); } Status link_file(const Path& src, const Path& dest) override { - return _local_fs->link_file(fmt::format("{}/{}", kTestDir, src.str()), - fmt::format("{}/{}", kTestDir, dest.str())); + return _local_fs->link_file(fmt::format("{}/{}", kTestDir, src.string()), + fmt::format("{}/{}", kTestDir, dest.string())); } Status exists(const Path& path, bool* res) const override { - return _local_fs->exists(fmt::format("{}/{}", kTestDir, path.str()), res); + return _local_fs->exists(fmt::format("{}/{}", kTestDir, path.string()), res); } Status file_size(const Path& path, size_t* file_size) const override { - return _local_fs->file_size(fmt::format("{}/{}", kTestDir, path.str()), file_size); + return _local_fs->file_size(fmt::format("{}/{}", kTestDir, path.string()), file_size); } Status list(const Path& path, std::vector* files) override { std::vector local_paths; - RETURN_IF_ERROR(_local_fs->list(fmt::format("{}/{}", kTestDir, path.str()), + RETURN_IF_ERROR(_local_fs->list(fmt::format("{}/{}", kTestDir, path.string()), &local_paths)); for (Path path : local_paths) { - files->emplace_back(path.str().substr(kTestDir.size() + 1)); + files->emplace_back(path.string().substr(kTestDir.size() + 1)); } return Status::OK(); } @@ -166,7 +166,7 @@ class TabletCooldownTest : public testing::Test { return Status::OK(); } private: - std::shared_ptr _local_fs; + std::shared_ptr _local_fs; }; public: From b416b87d510e11dd1fd79b20aba0d2ac5eb17f30 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 19:41:33 +0800 Subject: [PATCH 18/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 78a7b0ef8ee903..23abdd2de85696 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -23,9 +23,11 @@ #include "common/status.h" #include "exec/tablet_info.h" #include "gen_cpp/internal_service.pb.h" -#include "io/file_writer.h" +#include "io/fs/file_writer.h" +#include "io/fs/local_file_writer.h" #include "io/fs/s3_file_system.h" #include "io/fs/remote_file_system.h" +#include "io/fs/local_file_system.h" #include "olap/delta_writer.h" #include "olap/rowset/beta_rowset.h" #include "olap/storage_engine.h" @@ -55,7 +57,7 @@ class TabletCooldownTest : public testing::Test { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) { _local_file_writer = std::make_unique(fmt::format( - "{}/{}", kTestDir, _path.string())); + "{}/{}", kTestDir, _path.string()), 0); } ~FileWriterMock() {} From 68a5eef903989276f67204ef6a21db61f629720c Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Thu, 2 Mar 2023 19:49:22 +0800 Subject: [PATCH 19/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 36 +++++++++++++++++---------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 23abdd2de85696..481e4ecd50b783 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -57,7 +57,7 @@ class TabletCooldownTest : public testing::Test { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) { _local_file_writer = std::make_unique(fmt::format( - "{}/{}", kTestDir, _path.string()), 0); + "{}/{}", config::storage_root_path, _path.string()), 0); } ~FileWriterMock() {} @@ -96,7 +96,7 @@ class TabletCooldownTest : public testing::Test { class RemoteFileSystemMock : public io::RemoteFileSystem { RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) : RemoteFileSystem(std::move(root_path), std::move(id), type) { - _local_fs = io::LocalFileSystem::create(fmt::format("{}/{}", kTestDir, + _local_fs = io::LocalFileSystem::create(fmt::format("{}/{}", config::storage_root_path, _root_path.string())); } ~RemoteFileSystemMock() override {} @@ -107,40 +107,48 @@ class TabletCooldownTest : public testing::Test { } Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return _local_fs->open_file(fmt::format("{}/{}", kTestDir, path.string()), reader, io_ctx); + return _local_fs->open_file(fmt::format("{}/{}", config::storage_root_path, + path.string()), reader, io_ctx); } Status delete_file(const Path& path) override { - return _local_fs->delete_file(fmt::format("{}/{}", kTestDir, path.string())); + return _local_fs->delete_file(fmt::format("{}/{}", config::storage_root_path, + path.string())); } Status create_directory(const Path& path) override { - return _local_fs->create_directory(fmt::format("{}/{}", kTestDir, path.string())); + return _local_fs->create_directory(fmt::format("{}/{}", config::storage_root_path, + path.string())); } Status delete_directory(const Path& path) override { - return _local_fs->delete_directory(fmt::format("{}/{}", kTestDir, path.string())); + return _local_fs->delete_directory(fmt::format("{}/{}", config::storage_root_path, + path.string())); } Status link_file(const Path& src, const Path& dest) override { - return _local_fs->link_file(fmt::format("{}/{}", kTestDir, src.string()), - fmt::format("{}/{}", kTestDir, dest.string())); + return _local_fs->link_file(fmt::format("{}/{}", config::storage_root_path, + src.string()), + fmt::format("{}/{}", config::storage_root_path, + dest.string())); } Status exists(const Path& path, bool* res) const override { - return _local_fs->exists(fmt::format("{}/{}", kTestDir, path.string()), res); + return _local_fs->exists(fmt::format("{}/{}", config::storage_root_path, path.string()), + res); } Status file_size(const Path& path, size_t* file_size) const override { - return _local_fs->file_size(fmt::format("{}/{}", kTestDir, path.string()), file_size); + return _local_fs->file_size(fmt::format("{}/{}", config::storage_root_path, + path.string()), file_size); } Status list(const Path& path, std::vector* files) override { std::vector local_paths; - RETURN_IF_ERROR(_local_fs->list(fmt::format("{}/{}", kTestDir, path.string()), - &local_paths)); + RETURN_IF_ERROR(_local_fs->list(fmt::format("{}/{}", config::storage_root_path, + path.string()), &local_paths)); for (Path path : local_paths) { - files->emplace_back(path.string().substr(kTestDir.size() + 1)); + files->emplace_back(path.string().substr(config::storage_root_path.size() + 1)); } return Status::OK(); } @@ -191,6 +199,8 @@ class TabletCooldownTest : public testing::Test { FileUtils::remove_all(config::storage_root_path); FileUtils::create_dir(config::storage_root_path); + FileUtils::create_dir(fmt::format("{}/data/{}/{}.meta", config::storage_root_path, + kTabletId, kReplicaId); std::vector paths {{config::storage_root_path, -1}}; From 2982d2c396b0ba4d3a17079d6be8da8af5608d47 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 14:41:40 +0800 Subject: [PATCH 20/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 481e4ecd50b783..3925faf284b698 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -56,8 +56,8 @@ class TabletCooldownTest : public testing::Test { class FileWriterMock : public io::FileWriter { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) { - _local_file_writer = std::make_unique(fmt::format( - "{}/{}", config::storage_root_path, _path.string()), 0); + io::global_local_filesystem()->create_file(fmt::format( + "{}/{}", config::storage_root_path, _path.string()), &_local_file_writer); } ~FileWriterMock() {} @@ -90,7 +90,7 @@ class TabletCooldownTest : public testing::Test { io::FileSystemSPtr fs() const override { return s_fs; } private: - std::unique_ptr _local_file_writer; + std::unique_ptr _local_file_writer; }; class RemoteFileSystemMock : public io::RemoteFileSystem { @@ -199,8 +199,7 @@ class TabletCooldownTest : public testing::Test { FileUtils::remove_all(config::storage_root_path); FileUtils::create_dir(config::storage_root_path); - FileUtils::create_dir(fmt::format("{}/data/{}/{}.meta", config::storage_root_path, - kTabletId, kReplicaId); + FileUtils::create_dir(fmt::format("{}/data/{}", config::storage_root_path, kTabletId)); std::vector paths {{config::storage_root_path, -1}}; From 8883d26c9d18de8589cee288369440d1ef79441e Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 14:48:56 +0800 Subject: [PATCH 21/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 3925faf284b698..39cd85051ef5be 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -154,13 +154,14 @@ class TabletCooldownTest : public testing::Test { } Status upload(const Path& local_path, const Path& dest_path) override { - return link_file(local_path, dest_path); + return _local_fs->link_file(local_path.string(), fmt::format( + "{}/{}", config::storage_root_path, dest_path.string())); } Status batch_upload(const std::vector& local_paths, const std::vector& dest_paths) override { for (int i = 0; i < local_paths.size(); ++i) { - RETURN_IF_ERROR(link_file(local_paths[i], dest_paths[i])); + RETURN_IF_ERROR(upload(local_paths[i], dest_paths[i])); } return Status::OK(); } From 4d074914c9b84b568c6d5c627ef51138fb6db9b1 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 15:07:03 +0800 Subject: [PATCH 22/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 48 +++++++++++++-------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 39cd85051ef5be..fb42c82440a5aa 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -52,12 +52,15 @@ using io::Path; static io::FileSystemSPtr s_fs; +static std::string get_remote_path(const Path& path) { + return fmt::format("{}/remote/{}", config::storage_root_path, path.string()); +} + class TabletCooldownTest : public testing::Test { class FileWriterMock : public io::FileWriter { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) { - io::global_local_filesystem()->create_file(fmt::format( - "{}/{}", config::storage_root_path, _path.string()), &_local_file_writer); + io::global_local_filesystem()->create_file(get_remote_path(_path), &_local_file_writer); } ~FileWriterMock() {} @@ -96,8 +99,7 @@ class TabletCooldownTest : public testing::Test { class RemoteFileSystemMock : public io::RemoteFileSystem { RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) : RemoteFileSystem(std::move(root_path), std::move(id), type) { - _local_fs = io::LocalFileSystem::create(fmt::format("{}/{}", config::storage_root_path, - _root_path.string())); + _local_fs = io::LocalFileSystem::create(get_remote_path(_root_path)); } ~RemoteFileSystemMock() override {} @@ -107,46 +109,36 @@ class TabletCooldownTest : public testing::Test { } Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return _local_fs->open_file(fmt::format("{}/{}", config::storage_root_path, - path.string()), reader, io_ctx); + return _local_fs->open_file(get_remote_path(path), reader, io_ctx); } Status delete_file(const Path& path) override { - return _local_fs->delete_file(fmt::format("{}/{}", config::storage_root_path, - path.string())); + return _local_fs->delete_file(get_remote_path(path)); } Status create_directory(const Path& path) override { - return _local_fs->create_directory(fmt::format("{}/{}", config::storage_root_path, - path.string())); + return _local_fs->create_directory(get_remote_path(path)); } Status delete_directory(const Path& path) override { - return _local_fs->delete_directory(fmt::format("{}/{}", config::storage_root_path, - path.string())); + return _local_fs->delete_directory(get_remote_path(path)); } Status link_file(const Path& src, const Path& dest) override { - return _local_fs->link_file(fmt::format("{}/{}", config::storage_root_path, - src.string()), - fmt::format("{}/{}", config::storage_root_path, - dest.string())); + return _local_fs->link_file(get_remote_path(src), get_remote_path(dest)); } Status exists(const Path& path, bool* res) const override { - return _local_fs->exists(fmt::format("{}/{}", config::storage_root_path, path.string()), - res); + return _local_fs->exists(get_remote_path(path), res); } Status file_size(const Path& path, size_t* file_size) const override { - return _local_fs->file_size(fmt::format("{}/{}", config::storage_root_path, - path.string()), file_size); + return _local_fs->file_size(get_remote_path(path), file_size); } Status list(const Path& path, std::vector* files) override { std::vector local_paths; - RETURN_IF_ERROR(_local_fs->list(fmt::format("{}/{}", config::storage_root_path, - path.string()), &local_paths)); + RETURN_IF_ERROR(_local_fs->list(get_remote_path(path), &local_paths)); for (Path path : local_paths) { files->emplace_back(path.string().substr(config::storage_root_path.size() + 1)); } @@ -154,8 +146,7 @@ class TabletCooldownTest : public testing::Test { } Status upload(const Path& local_path, const Path& dest_path) override { - return _local_fs->link_file(local_path.string(), fmt::format( - "{}/{}", config::storage_root_path, dest_path.string())); + return _local_fs->link_file(local_path.string(), get_remote_path(dest_path)); } Status batch_upload(const std::vector& local_paths, @@ -200,7 +191,7 @@ class TabletCooldownTest : public testing::Test { FileUtils::remove_all(config::storage_root_path); FileUtils::create_dir(config::storage_root_path); - FileUtils::create_dir(fmt::format("{}/data/{}", config::storage_root_path, kTabletId)); + FileUtils::create_dir(get_remote_path(fmt::format("data/{}", kTabletId))); std::vector paths {{config::storage_root_path, -1}}; @@ -367,6 +358,13 @@ TEST_F(TabletCooldownTest, normal) { ASSERT_EQ(Status::OK(), st); auto rs = tablet->get_rowset_by_version({2, 2}); ASSERT_FALSE(rs->is_local()); + + // test read + ASSERT_EQ(Status::OK(), st); + std::vector segments; + st = std::static_pointer_cast(rs)->load_segments(&segments); + ASSERT_EQ(Status::OK(), st); + ASSERT_EQ(segments.size(), 1); } } // namespace doris From 40f256f046b5f1a16fdd92b6596c35b69b44816c Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 15:21:04 +0800 Subject: [PATCH 23/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index fb42c82440a5aa..25b117f3beaea3 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -57,13 +57,14 @@ static std::string get_remote_path(const Path& path) { } class TabletCooldownTest : public testing::Test { +public: class FileWriterMock : public io::FileWriter { public: FileWriterMock(Path path) : io::FileWriter(std::move(path)) { io::global_local_filesystem()->create_file(get_remote_path(_path), &_local_file_writer); } - ~FileWriterMock() {} + ~FileWriterMock() = default; Status close() override { return _local_file_writer->close(); @@ -101,10 +102,11 @@ class TabletCooldownTest : public testing::Test { : RemoteFileSystem(std::move(root_path), std::move(id), type) { _local_fs = io::LocalFileSystem::create(get_remote_path(_root_path)); } - ~RemoteFileSystemMock() override {} + ~RemoteFileSystemMock() = default; Status create_file(const Path& path, io::FileWriterPtr* writer) override { - *writer = std::make_unique(path); + Path fs_path = path; + *writer = std::make_unique(fs_path); return Status::OK(); } From b95edb73ee7c53d2a73f99d2561564b0492e1665 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 15:26:32 +0800 Subject: [PATCH 24/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 25b117f3beaea3..02f6d813053983 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -66,17 +66,11 @@ class TabletCooldownTest : public testing::Test { ~FileWriterMock() = default; - Status close() override { - return _local_file_writer->close(); - } + Status close() override { return _local_file_writer->close(); } - Status abort() override { - return _local_file_writer->abort(); - } + Status abort() override { return _local_file_writer->abort(); } - Status append(const Slice& data) override { - return _local_file_writer->append(data); - } + Status append(const Slice& data) override { return _local_file_writer->append(data); } Status appendv(const Slice* data, size_t data_cnt) override { return _local_file_writer->appendv(data, data_cnt); @@ -86,9 +80,7 @@ class TabletCooldownTest : public testing::Test { return _local_file_writer->write_at(offset, data); } - Status finalize() override { - return _local_file_writer->finalize(); - } + Status finalize() override { return _local_file_writer->finalize(); } size_t bytes_appended() const override { return _local_file_writer->bytes_appended(); } @@ -166,9 +158,7 @@ class TabletCooldownTest : public testing::Test { return Status::OK(); } - Status connect() override { - return Status::OK(); - } + Status connect() override { return Status::OK(); } private: std::shared_ptr _local_fs; }; @@ -288,7 +278,7 @@ TEST_F(TabletCooldownTest, normal) { PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); - WriteRequest write_req = {kTabletId, kSchemaHash, WriteType::LOAD, 20003, 30003, + WriteRequest write_req = {kTabletId, kSchemaHash, WriteType::LOAD, 20003, 30003, load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); From cd0a776bf342d8686f318e5080dc9e041dce861a Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 15:30:09 +0800 Subject: [PATCH 25/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 02f6d813053983..8f3e1a00990075 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -24,10 +24,10 @@ #include "exec/tablet_info.h" #include "gen_cpp/internal_service.pb.h" #include "io/fs/file_writer.h" +#include "io/fs/local_file_system.h" #include "io/fs/local_file_writer.h" -#include "io/fs/s3_file_system.h" #include "io/fs/remote_file_system.h" -#include "io/fs/local_file_system.h" +#include "io/fs/s3_file_system.h" #include "olap/delta_writer.h" #include "olap/rowset/beta_rowset.h" #include "olap/storage_engine.h" @@ -278,7 +278,7 @@ TEST_F(TabletCooldownTest, normal) { PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); - WriteRequest write_req = {kTabletId, kSchemaHash, WriteType::LOAD, 20003, 30003, + WriteRequest write_req = {kTabletId, kSchemaHash, WriteType::LOAD, 20003, 30003, load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); From b309226a2105ddc26561fd2b641983754ec11c68 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 15:33:29 +0800 Subject: [PATCH 26/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 8f3e1a00990075..bfda1b92fe4e79 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -64,7 +64,7 @@ class TabletCooldownTest : public testing::Test { io::global_local_filesystem()->create_file(get_remote_path(_path), &_local_file_writer); } - ~FileWriterMock() = default; + ~FileWriterMock() override = default; Status close() override { return _local_file_writer->close(); } @@ -85,16 +85,18 @@ class TabletCooldownTest : public testing::Test { size_t bytes_appended() const override { return _local_file_writer->bytes_appended(); } io::FileSystemSPtr fs() const override { return s_fs; } + private: std::unique_ptr _local_file_writer; }; class RemoteFileSystemMock : public io::RemoteFileSystem { + public: RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) : RemoteFileSystem(std::move(root_path), std::move(id), type) { _local_fs = io::LocalFileSystem::create(get_remote_path(_root_path)); } - ~RemoteFileSystemMock() = default; + ~RemoteFileSystemMock() override = default; Status create_file(const Path& path, io::FileWriterPtr* writer) override { Path fs_path = path; @@ -159,11 +161,11 @@ class TabletCooldownTest : public testing::Test { } Status connect() override { return Status::OK(); } + private: std::shared_ptr _local_fs; }; -public: static void SetUpTestSuite() { s_fs.reset(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), io::FileSystemType::S3)); @@ -279,7 +281,7 @@ TEST_F(TabletCooldownTest, normal) { load_id.set_hi(0); load_id.set_lo(0); WriteRequest write_req = {kTabletId, kSchemaHash, WriteType::LOAD, 20003, 30003, - load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; + load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); ASSERT_NE(delta_writer, nullptr); From 86abbb44e8ab72b9be73c004e18fd3bf85c0b0ae Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 15:40:03 +0800 Subject: [PATCH 27/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 160 +++++++++++++------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index bfda1b92fe4e79..d9ea8445af6667 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -56,116 +56,116 @@ static std::string get_remote_path(const Path& path) { return fmt::format("{}/remote/{}", config::storage_root_path, path.string()); } -class TabletCooldownTest : public testing::Test { +class FileWriterMock : public io::FileWriter { public: - class FileWriterMock : public io::FileWriter { - public: - FileWriterMock(Path path) : io::FileWriter(std::move(path)) { - io::global_local_filesystem()->create_file(get_remote_path(_path), &_local_file_writer); - } + FileWriterMock(Path path) : io::FileWriter(std::move(path)) { + io::global_local_filesystem()->create_file(get_remote_path(_path), &_local_file_writer); + } - ~FileWriterMock() override = default; + ~FileWriterMock() override = default; - Status close() override { return _local_file_writer->close(); } + Status close() override { return _local_file_writer->close(); } - Status abort() override { return _local_file_writer->abort(); } + Status abort() override { return _local_file_writer->abort(); } - Status append(const Slice& data) override { return _local_file_writer->append(data); } + Status append(const Slice& data) override { return _local_file_writer->append(data); } - Status appendv(const Slice* data, size_t data_cnt) override { - return _local_file_writer->appendv(data, data_cnt); - } + Status appendv(const Slice* data, size_t data_cnt) override { + return _local_file_writer->appendv(data, data_cnt); + } - Status write_at(size_t offset, const Slice& data) override { - return _local_file_writer->write_at(offset, data); - } + Status write_at(size_t offset, const Slice& data) override { + return _local_file_writer->write_at(offset, data); + } - Status finalize() override { return _local_file_writer->finalize(); } + Status finalize() override { return _local_file_writer->finalize(); } - size_t bytes_appended() const override { return _local_file_writer->bytes_appended(); } + size_t bytes_appended() const override { return _local_file_writer->bytes_appended(); } - io::FileSystemSPtr fs() const override { return s_fs; } + io::FileSystemSPtr fs() const override { return s_fs; } - private: - std::unique_ptr _local_file_writer; - }; +private: + std::unique_ptr _local_file_writer; +}; - class RemoteFileSystemMock : public io::RemoteFileSystem { - public: - RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) - : RemoteFileSystem(std::move(root_path), std::move(id), type) { - _local_fs = io::LocalFileSystem::create(get_remote_path(_root_path)); - } - ~RemoteFileSystemMock() override = default; +class RemoteFileSystemMock : public io::RemoteFileSystem { +public: + RemoteFileSystemMock(Path root_path, std::string&& id, io::FileSystemType type) + : RemoteFileSystem(std::move(root_path), std::move(id), type) { + _local_fs = io::LocalFileSystem::create(get_remote_path(_root_path)); + } + ~RemoteFileSystemMock() override = default; - Status create_file(const Path& path, io::FileWriterPtr* writer) override { - Path fs_path = path; - *writer = std::make_unique(fs_path); - return Status::OK(); - } + Status create_file(const Path& path, io::FileWriterPtr* writer) override { + Path fs_path = path; + *writer = std::make_unique(fs_path); + return Status::OK(); + } - Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { - return _local_fs->open_file(get_remote_path(path), reader, io_ctx); - } + Status open_file(const Path& path, io::FileReaderSPtr* reader, IOContext* io_ctx) override { + return _local_fs->open_file(get_remote_path(path), reader, io_ctx); + } - Status delete_file(const Path& path) override { - return _local_fs->delete_file(get_remote_path(path)); - } + Status delete_file(const Path& path) override { + return _local_fs->delete_file(get_remote_path(path)); + } - Status create_directory(const Path& path) override { - return _local_fs->create_directory(get_remote_path(path)); - } + Status create_directory(const Path& path) override { + return _local_fs->create_directory(get_remote_path(path)); + } - Status delete_directory(const Path& path) override { - return _local_fs->delete_directory(get_remote_path(path)); - } + Status delete_directory(const Path& path) override { + return _local_fs->delete_directory(get_remote_path(path)); + } - Status link_file(const Path& src, const Path& dest) override { - return _local_fs->link_file(get_remote_path(src), get_remote_path(dest)); - } + Status link_file(const Path& src, const Path& dest) override { + return _local_fs->link_file(get_remote_path(src), get_remote_path(dest)); + } - Status exists(const Path& path, bool* res) const override { - return _local_fs->exists(get_remote_path(path), res); - } + Status exists(const Path& path, bool* res) const override { + return _local_fs->exists(get_remote_path(path), res); + } - Status file_size(const Path& path, size_t* file_size) const override { - return _local_fs->file_size(get_remote_path(path), file_size); - } + Status file_size(const Path& path, size_t* file_size) const override { + return _local_fs->file_size(get_remote_path(path), file_size); + } - Status list(const Path& path, std::vector* files) override { - std::vector local_paths; - RETURN_IF_ERROR(_local_fs->list(get_remote_path(path), &local_paths)); - for (Path path : local_paths) { - files->emplace_back(path.string().substr(config::storage_root_path.size() + 1)); - } - return Status::OK(); + Status list(const Path& path, std::vector* files) override { + std::vector local_paths; + RETURN_IF_ERROR(_local_fs->list(get_remote_path(path), &local_paths)); + for (Path path : local_paths) { + files->emplace_back(path.string().substr(config::storage_root_path.size() + 1)); } + return Status::OK(); + } - Status upload(const Path& local_path, const Path& dest_path) override { - return _local_fs->link_file(local_path.string(), get_remote_path(dest_path)); - } + Status upload(const Path& local_path, const Path& dest_path) override { + return _local_fs->link_file(local_path.string(), get_remote_path(dest_path)); + } - Status batch_upload(const std::vector& local_paths, - const std::vector& dest_paths) override { - for (int i = 0; i < local_paths.size(); ++i) { - RETURN_IF_ERROR(upload(local_paths[i], dest_paths[i])); - } - return Status::OK(); + Status batch_upload(const std::vector& local_paths, + const std::vector& dest_paths) override { + for (int i = 0; i < local_paths.size(); ++i) { + RETURN_IF_ERROR(upload(local_paths[i], dest_paths[i])); } + return Status::OK(); + } - Status batch_delete(const std::vector& paths) override { - for (int i = 0; i < paths.size(); ++i) { - RETURN_IF_ERROR(delete_file(paths[i])); - } - return Status::OK(); + Status batch_delete(const std::vector& paths) override { + for (int i = 0; i < paths.size(); ++i) { + RETURN_IF_ERROR(delete_file(paths[i])); } + return Status::OK(); + } - Status connect() override { return Status::OK(); } + Status connect() override { return Status::OK(); } - private: - std::shared_ptr _local_fs; - }; +private: + std::shared_ptr _local_fs; +}; +class TabletCooldownTest : public testing::Test { +public: static void SetUpTestSuite() { s_fs.reset(new RemoteFileSystemMock("test_path", std::to_string(kResourceId), io::FileSystemType::S3)); From 63c0d8854d5200c6101d623b3eceb0e29c7e103b Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 18:20:45 +0800 Subject: [PATCH 28/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 89 ++++++++++++++++++++------- 1 file changed, 66 insertions(+), 23 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index d9ea8445af6667..5caaf2ef3f07d6 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -45,8 +45,16 @@ static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; static constexpr int64_t kTabletId = 10005; +static constexpr int64_t kTabletId2 = 10006; static constexpr int64_t kReplicaId = 10009; -static constexpr int64_t kSchemaHash = 270068377; +static constexpr int32_t kSchemaHash = 270068377; +static constexpr int64_t kReplicaId2 = 10010; +static constexpr int32_t kSchemaHash2 = 270068381; + +static constexpr int32_t kTxnId = 20003; +static constexpr int32_t kPartitionId = 30003; +static constexpr int32_t kTxnId2 = 40003; +static constexpr int32_t kPartitionId2 = 50003; using io::Path; @@ -186,6 +194,7 @@ class TabletCooldownTest : public testing::Test { FileUtils::remove_all(config::storage_root_path); FileUtils::create_dir(config::storage_root_path); FileUtils::create_dir(get_remote_path(fmt::format("data/{}", kTabletId))); + FileUtils::create_dir(get_remote_path(fmt::format("data/{}", kTabletId2))); std::vector paths {{config::storage_root_path, -1}}; @@ -261,12 +270,13 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() { return desc_tbl_builder.desc_tbl(); } -TEST_F(TabletCooldownTest, normal) { +void createTablet(StorageEngine* engine, TabletSharedPtr* tablet, int64_t replica_id, int32_t schema_hash, + int64_t tablet_id, int64_t txn_id, int64_t partition_id) { // create tablet TCreateTabletReq request; - create_tablet_request_with_sequence_col(kTabletId, kSchemaHash, &request); - request.__set_replica_id(kReplicaId); - Status st = k_engine->create_tablet(request); + create_tablet_request_with_sequence_col(tablet_id, schema_hash, &request); + request.__set_replica_id(replica_id); + Status st = engine->create_tablet(request); ASSERT_EQ(Status::OK(), st); TDescriptorTable tdesc_tbl = create_descriptor_tablet_with_sequence_col(); @@ -280,7 +290,8 @@ TEST_F(TabletCooldownTest, normal) { PUniqueId load_id; load_id.set_hi(0); load_id.set_lo(0); - WriteRequest write_req = {kTabletId, kSchemaHash, WriteType::LOAD, 20003, 30003, + + WriteRequest write_req = {tablet_id, schema_hash, WriteType::LOAD, txn_id, partition_id, load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); @@ -320,37 +331,43 @@ TEST_F(TabletCooldownTest, normal) { delete delta_writer; // publish version success - TabletSharedPtr tablet = - k_engine->tablet_manager()->get_tablet(write_req.tablet_id, write_req.schema_hash); - OlapMeta* meta = tablet->data_dir()->get_meta(); + *tablet = engine->tablet_manager()->get_tablet(write_req.tablet_id, write_req.schema_hash); + OlapMeta* meta = (*tablet)->data_dir()->get_meta(); Version version; - version.first = tablet->rowset_with_max_version()->end_version() + 1; - version.second = tablet->rowset_with_max_version()->end_version() + 1; + version.first = (*tablet)->rowset_with_max_version()->end_version() + 1; + version.second = (*tablet)->rowset_with_max_version()->end_version() + 1; std::map tablet_related_rs; - StorageEngine::instance()->txn_manager()->get_txn_related_tablets( + engine->txn_manager()->get_txn_related_tablets( write_req.txn_id, write_req.partition_id, &tablet_related_rs); for (auto& tablet_rs : tablet_related_rs) { RowsetSharedPtr rowset = tablet_rs.second; - st = k_engine->txn_manager()->publish_txn(meta, write_req.partition_id, write_req.txn_id, - tablet->tablet_id(), tablet->schema_hash(), - tablet->tablet_uid(), version); + st = engine->txn_manager()->publish_txn(meta, write_req.partition_id, write_req.txn_id, + (*tablet)->tablet_id(), (*tablet)->schema_hash(), + (*tablet)->tablet_uid(), version); ASSERT_EQ(Status::OK(), st); - st = tablet->add_inc_rowset(rowset); + st = (*tablet)->add_inc_rowset(rowset); ASSERT_EQ(Status::OK(), st); } - EXPECT_EQ(1, tablet->num_rows()); + EXPECT_EQ(1, (*tablet)->num_rows()); + +} +TEST_F(TabletCooldownTest, normal) { + TabletSharedPtr tablet1; + TabletSharedPtr tablet2; + createTablet(k_engine, &tablet1, kReplicaId, kSchemaHash, kTabletId, kTxnId, kPartitionId); + createTablet(k_engine, &tablet2, kReplicaId2, kSchemaHash2, kTabletId2, kTxnId2, kPartitionId2); // test cooldown - tablet->set_storage_policy_id(kStoragePolicyId); - st = tablet->cooldown(); // rowset [0-1] + tablet1->set_storage_policy_id(kStoragePolicyId); + Status st = tablet1->cooldown(); // rowset [0-1] ASSERT_NE(Status::OK(), st); - tablet->update_cooldown_conf(1, kReplicaId); + tablet1->update_cooldown_conf(1, kReplicaId); // cooldown for upload node - st = tablet->cooldown(); // rowset [0-1] + st = tablet1->cooldown(); // rowset [0-1] ASSERT_EQ(Status::OK(), st); - st = tablet->cooldown(); // rowset [2-2] + st = tablet1->cooldown(); // rowset [2-2] ASSERT_EQ(Status::OK(), st); - auto rs = tablet->get_rowset_by_version({2, 2}); + auto rs = tablet1->get_rowset_by_version({2, 2}); ASSERT_FALSE(rs->is_local()); // test read @@ -359,6 +376,32 @@ TEST_F(TabletCooldownTest, normal) { st = std::static_pointer_cast(rs)->load_segments(&segments); ASSERT_EQ(Status::OK(), st); ASSERT_EQ(segments.size(), 1); + + st = io::global_local_filesystem()->link_file( + get_remote_path(fmt::format("data/{}/{}.meta", kTabletId, kReplicaId)), + get_remote_path(fmt::format("data/{}/{}.meta", kTabletId2, kReplicaId))); + ASSERT_EQ(Status::OK(), st); + // follow cooldown + tablet2->set_storage_policy_id(kStoragePolicyId); + tablet2->update_cooldown_conf(1, 111111111); + st = tablet2->cooldown(); // rowset [0-1] + ASSERT_NE(Status::OK(), st); + tablet2->update_cooldown_conf(1, kReplicaId); + st = tablet2->cooldown(); // rowset [0-1] + ASSERT_NE(Status::OK(), st); + tablet2->update_cooldown_conf(2, kReplicaId); + st = tablet2->cooldown(); // rowset [0-1] + ASSERT_EQ(Status::OK(), st); + auto rs2 = tablet2->get_rowset_by_version({2, 2}); + ASSERT_FALSE(rs2->is_local()); + + // test read tablet2 + ASSERT_EQ(Status::OK(), st); + std::vector segments2; + st = std::static_pointer_cast(rs2)->load_segments(&segments2); + ASSERT_EQ(Status::OK(), st); + ASSERT_EQ(segments2.size(), 1); + } } // namespace doris From bb68e7d5e604ffbe88e267faa0eb1dcf5cdad8af Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Fri, 3 Mar 2023 18:23:33 +0800 Subject: [PATCH 29/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 5caaf2ef3f07d6..0ce5b6aec5ce89 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -270,8 +270,8 @@ static TDescriptorTable create_descriptor_tablet_with_sequence_col() { return desc_tbl_builder.desc_tbl(); } -void createTablet(StorageEngine* engine, TabletSharedPtr* tablet, int64_t replica_id, int32_t schema_hash, - int64_t tablet_id, int64_t txn_id, int64_t partition_id) { +void createTablet(StorageEngine* engine, TabletSharedPtr* tablet, int64_t replica_id, + int32_t schema_hash, int64_t tablet_id, int64_t txn_id, int64_t partition_id) { // create tablet TCreateTabletReq request; create_tablet_request_with_sequence_col(tablet_id, schema_hash, &request); @@ -292,7 +292,7 @@ void createTablet(StorageEngine* engine, TabletSharedPtr* tablet, int64_t replic load_id.set_lo(0); WriteRequest write_req = {tablet_id, schema_hash, WriteType::LOAD, txn_id, partition_id, - load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; + load_id, tuple_desc, &(tuple_desc->slots()), false, ¶m}; DeltaWriter* delta_writer = nullptr; DeltaWriter::open(&write_req, &delta_writer); ASSERT_NE(delta_writer, nullptr); @@ -337,8 +337,8 @@ void createTablet(StorageEngine* engine, TabletSharedPtr* tablet, int64_t replic version.first = (*tablet)->rowset_with_max_version()->end_version() + 1; version.second = (*tablet)->rowset_with_max_version()->end_version() + 1; std::map tablet_related_rs; - engine->txn_manager()->get_txn_related_tablets( - write_req.txn_id, write_req.partition_id, &tablet_related_rs); + engine->txn_manager()->get_txn_related_tablets(write_req.txn_id, write_req.partition_id, + &tablet_related_rs); for (auto& tablet_rs : tablet_related_rs) { RowsetSharedPtr rowset = tablet_rs.second; st = engine->txn_manager()->publish_txn(meta, write_req.partition_id, write_req.txn_id, @@ -349,7 +349,6 @@ void createTablet(StorageEngine* engine, TabletSharedPtr* tablet, int64_t replic ASSERT_EQ(Status::OK(), st); } EXPECT_EQ(1, (*tablet)->num_rows()); - } TEST_F(TabletCooldownTest, normal) { @@ -401,7 +400,6 @@ TEST_F(TabletCooldownTest, normal) { st = std::static_pointer_cast(rs2)->load_segments(&segments2); ASSERT_EQ(Status::OK(), st); ASSERT_EQ(segments2.size(), 1); - } } // namespace doris From db486ba750690584a170f3c3d6fe4f19bfa140d0 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 10:04:10 +0800 Subject: [PATCH 30/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 0ce5b6aec5ce89..a9510a40d8ae8f 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -41,7 +41,7 @@ namespace doris { static StorageEngine* k_engine = nullptr; -static const std::string kTestDir = "./ut_dir/tablet_cooldown_test"; +static const std::string kTestDir = "ut_dir/tablet_cooldown_test"; static constexpr int64_t kResourceId = 10000; static constexpr int64_t kStoragePolicyId = 10002; static constexpr int64_t kTabletId = 10005; From aa17de1766dcd5b8b26b5ce1aa6725931d08f9f7 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 11:47:19 +0800 Subject: [PATCH 31/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index a9510a40d8ae8f..51880666c543cb 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -141,8 +141,8 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { Status list(const Path& path, std::vector* files) override { std::vector local_paths; RETURN_IF_ERROR(_local_fs->list(get_remote_path(path), &local_paths)); - for (Path path : local_paths) { - files->emplace_back(path.string().substr(config::storage_root_path.size() + 1)); + for (Path local_path : local_paths) { + files->emplace_back(fmt::format("remote/{}/{}", path, local_path.string())); } return Status::OK(); } @@ -376,6 +376,26 @@ TEST_F(TabletCooldownTest, normal) { ASSERT_EQ(Status::OK(), st); ASSERT_EQ(segments.size(), 1); + std::vector files; + ASSERT_EQ(Status::OK(), io::global_local_filesystem()->list(get_remote_path("data"), &files)); + printf("list: %s\n", get_remote_path("data").c_str()); + for (auto file : files) { + printf("file: %s\n", file.string().c_str()); + } + files.clear(); + ASSERT_EQ(Status::OK(), io::global_local_filesystem()->list(get_remote_path(fmt::format("data/{}", kTabletId)), &files)); + printf("list: %s\n", get_remote_path(fmt::format("data/{}", kTabletId)).c_str()); + for (auto file : files) { + printf("file: %s\n", file.string().c_str()); + } + files.clear(); + ASSERT_EQ(Status::OK(), io::global_local_filesystem()->list(get_remote_path(fmt::format("data/{}", kTabletId2)), &files)); + printf("list: %s\n", get_remote_path(fmt::format("data/{}", kTabletId2)).c_str()); + for (auto file : files) { + printf("file: %s\n", file.string().c_str()); + } + files.clear(); + st = io::global_local_filesystem()->link_file( get_remote_path(fmt::format("data/{}/{}.meta", kTabletId, kReplicaId)), get_remote_path(fmt::format("data/{}/{}.meta", kTabletId2, kReplicaId))); From 893fdf3136f33b88e9c936324bd669a0ce939381 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 11:50:38 +0800 Subject: [PATCH 32/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 51880666c543cb..3c2d0ddbf7848f 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -142,7 +142,7 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { std::vector local_paths; RETURN_IF_ERROR(_local_fs->list(get_remote_path(path), &local_paths)); for (Path local_path : local_paths) { - files->emplace_back(fmt::format("remote/{}/{}", path, local_path.string())); + files->emplace_back(fmt::format("remote/{}/{}", path.string(), local_path.string())); } return Status::OK(); } From 710881c63f9ac04373a3717ad0010ff5a98a488b Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 14:47:23 +0800 Subject: [PATCH 33/36] add ut for cooldown on be --- be/src/olap/rowset/beta_rowset.cpp | 1 + be/test/olap/tablet_cooldown_test.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 26d4b0b7ce2fdf..9bc244935efe87 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -76,6 +76,7 @@ std::string BetaRowset::remote_tablet_path(int64_t tablet_id) { std::string BetaRowset::remote_tablet_meta_path(int64_t tablet_id, int64_t replica_id) { // data/{tablet_id}/{replica_id}.meta + printf("remote_tablet_meta_path: %ld, %ld\n", tablet_id, replica_id); return fmt::format("{}/{}.meta", remote_tablet_path(tablet_id), replica_id); } diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index 3c2d0ddbf7848f..a9fe361ea06287 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -76,13 +76,18 @@ class FileWriterMock : public io::FileWriter { Status abort() override { return _local_file_writer->abort(); } - Status append(const Slice& data) override { return _local_file_writer->append(data); } + Status append(const Slice& data) override { + printf("append: %s\n", path.string().c_str()); + return _local_file_writer->append(data); + } Status appendv(const Slice* data, size_t data_cnt) override { + printf("appendv: %s\n", path.string().c_str()); return _local_file_writer->appendv(data, data_cnt); } Status write_at(size_t offset, const Slice& data) override { + printf("write_at: %s\n", path.string().c_str()); return _local_file_writer->write_at(offset, data); } @@ -105,6 +110,7 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { ~RemoteFileSystemMock() override = default; Status create_file(const Path& path, io::FileWriterPtr* writer) override { + printf("create_file: %s\n", path.string().c_str()); Path fs_path = path; *writer = std::make_unique(fs_path); return Status::OK(); From 8154c66faba62147162780b11d7fcf25706ab7d6 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 14:48:40 +0800 Subject: [PATCH 34/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index a9fe361ea06287..eacb656be62198 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -77,17 +77,17 @@ class FileWriterMock : public io::FileWriter { Status abort() override { return _local_file_writer->abort(); } Status append(const Slice& data) override { - printf("append: %s\n", path.string().c_str()); + printf("append: %s\n", _path.string().c_str()); return _local_file_writer->append(data); } Status appendv(const Slice* data, size_t data_cnt) override { - printf("appendv: %s\n", path.string().c_str()); + printf("appendv: %s\n", _path.string().c_str()); return _local_file_writer->appendv(data, data_cnt); } Status write_at(size_t offset, const Slice& data) override { - printf("write_at: %s\n", path.string().c_str()); + printf("write_at: %s\n", _path.string().c_str()); return _local_file_writer->write_at(offset, data); } From 3d56f622c1fde141ffce65f02624bbe5548695a8 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 15:01:50 +0800 Subject: [PATCH 35/36] add ut for cooldown on be --- be/src/olap/rowset/beta_rowset.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index 9bc244935efe87..26d4b0b7ce2fdf 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -76,7 +76,6 @@ std::string BetaRowset::remote_tablet_path(int64_t tablet_id) { std::string BetaRowset::remote_tablet_meta_path(int64_t tablet_id, int64_t replica_id) { // data/{tablet_id}/{replica_id}.meta - printf("remote_tablet_meta_path: %ld, %ld\n", tablet_id, replica_id); return fmt::format("{}/{}.meta", remote_tablet_path(tablet_id), replica_id); } From 7a411edd98a31b85a686cff4e4dab6c51f7522dd Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 15:09:09 +0800 Subject: [PATCH 36/36] add ut for cooldown on be --- be/test/olap/tablet_cooldown_test.cpp | 36 ++++----------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index eacb656be62198..b842f82d597a4a 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -76,18 +76,13 @@ class FileWriterMock : public io::FileWriter { Status abort() override { return _local_file_writer->abort(); } - Status append(const Slice& data) override { - printf("append: %s\n", _path.string().c_str()); - return _local_file_writer->append(data); - } + Status append(const Slice& data) override { return _local_file_writer->append(data); } Status appendv(const Slice* data, size_t data_cnt) override { - printf("appendv: %s\n", _path.string().c_str()); return _local_file_writer->appendv(data, data_cnt); } Status write_at(size_t offset, const Slice& data) override { - printf("write_at: %s\n", _path.string().c_str()); return _local_file_writer->write_at(offset, data); } @@ -110,7 +105,6 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { ~RemoteFileSystemMock() override = default; Status create_file(const Path& path, io::FileWriterPtr* writer) override { - printf("create_file: %s\n", path.string().c_str()); Path fs_path = path; *writer = std::make_unique(fs_path); return Status::OK(); @@ -147,8 +141,8 @@ class RemoteFileSystemMock : public io::RemoteFileSystem { Status list(const Path& path, std::vector* files) override { std::vector local_paths; RETURN_IF_ERROR(_local_fs->list(get_remote_path(path), &local_paths)); - for (Path local_path : local_paths) { - files->emplace_back(fmt::format("remote/{}/{}", path.string(), local_path.string())); + for (Path path : local_paths) { + files->emplace_back(path.string().substr(config::storage_root_path.size() + 1)); } return Status::OK(); } @@ -382,29 +376,9 @@ TEST_F(TabletCooldownTest, normal) { ASSERT_EQ(Status::OK(), st); ASSERT_EQ(segments.size(), 1); - std::vector files; - ASSERT_EQ(Status::OK(), io::global_local_filesystem()->list(get_remote_path("data"), &files)); - printf("list: %s\n", get_remote_path("data").c_str()); - for (auto file : files) { - printf("file: %s\n", file.string().c_str()); - } - files.clear(); - ASSERT_EQ(Status::OK(), io::global_local_filesystem()->list(get_remote_path(fmt::format("data/{}", kTabletId)), &files)); - printf("list: %s\n", get_remote_path(fmt::format("data/{}", kTabletId)).c_str()); - for (auto file : files) { - printf("file: %s\n", file.string().c_str()); - } - files.clear(); - ASSERT_EQ(Status::OK(), io::global_local_filesystem()->list(get_remote_path(fmt::format("data/{}", kTabletId2)), &files)); - printf("list: %s\n", get_remote_path(fmt::format("data/{}", kTabletId2)).c_str()); - for (auto file : files) { - printf("file: %s\n", file.string().c_str()); - } - files.clear(); - st = io::global_local_filesystem()->link_file( - get_remote_path(fmt::format("data/{}/{}.meta", kTabletId, kReplicaId)), - get_remote_path(fmt::format("data/{}/{}.meta", kTabletId2, kReplicaId))); + get_remote_path(fmt::format("data/{}/{}.{}.meta", kTabletId, kReplicaId, 1)), + get_remote_path(fmt::format("data/{}/{}.{}.meta", kTabletId2, kReplicaId, 2))); ASSERT_EQ(Status::OK(), st); // follow cooldown tablet2->set_storage_policy_id(kStoragePolicyId);