From 6e6fa72623fddfe120c401699e0bf6c8f386acfb Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Tue, 9 Apr 2024 10:12:20 -0700 Subject: [PATCH 01/11] Adopt device type Signed-off-by: Xiaoxi Chen --- conanfile.py | 2 +- src/include/homeobject/homeobject.hpp | 9 ++- src/lib/homestore_backend/hs_homeobject.cpp | 71 +++++++++++++++++---- src/lib/homestore_backend/hs_homeobject.hpp | 3 + src/lib/tests/fixture_app.hpp | 6 +- test_package/test_package.cpp | 4 +- 6 files changed, 76 insertions(+), 19 deletions(-) diff --git a/conanfile.py b/conanfile.py index 0a6e58b2e..2112f7ed7 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "1.0.16" + version = "1.1.1" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeReplication" topics = ("ebay") diff --git a/src/include/homeobject/homeobject.hpp b/src/include/homeobject/homeobject.hpp index 9870b1642..4c3d5f7d9 100644 --- a/src/include/homeobject/homeobject.hpp +++ b/src/include/homeobject/homeobject.hpp @@ -6,12 +6,19 @@ #include #include "common.hpp" +#include namespace homeobject { class BlobManager; class PGManager; class ShardManager; +ENUM(DevType, uint8_t, AUTO_DETECT = 1, HDD, NVME, UNSUPPORTED); +struct device_info_t { + explicit device_info_t(std::string name, DevType dtype = DevType::AUTO_DETECT) : path{std::filesystem::canonical(name)}, type{dtype} {} + std::filesystem::path path; + DevType type; +}; class HomeObjectApplication { public: @@ -19,7 +26,7 @@ class HomeObjectApplication { virtual bool spdk_mode() const = 0; virtual uint32_t threads() const = 0; - virtual std::list< std::filesystem::path > devices() const = 0; + virtual std::list< device_info_t > devices() const = 0; // Callback made after determining if a SvcId exists or not during initialization, will consume response virtual peer_id_t discover_svcid(std::optional< peer_id_t > const& found) const = 0; diff --git a/src/lib/homestore_backend/hs_homeobject.cpp b/src/lib/homestore_backend/hs_homeobject.cpp index 8cd9824ec..fb784f049 100644 --- a/src/lib/homestore_backend/hs_homeobject.cpp +++ b/src/lib/homestore_backend/hs_homeobject.cpp @@ -97,6 +97,13 @@ class HSReplApplication : public homestore::ReplApplication { // // This should assert if we can not initialize HomeStore. // +DevType HSHomeObject::get_device_type(string const& devname) { + const iomgr::drive_type dtype = iomgr::DriveInterface::get_drive_type(devname); + if (dtype == iomgr::drive_type::block_hdd || dtype == iomgr::drive_type::file_on_hdd) { return DevType::HDD; } + if (dtype == iomgr::drive_type::file_on_nvme || dtype == iomgr::drive_type::block_nvme) { return DevType::NVME; } + return DevType::UNSUPPORTED; +} + void HSHomeObject::init_homestore() { auto app = _application.lock(); RELEASE_ASSERT(app, "HomeObjectApplication lifetime unexpected!"); @@ -115,8 +122,25 @@ void HSHomeObject::init_homestore() { LOGI("Initialize and start HomeStore with app_mem_size = {}", homestore::in_bytes(app_mem_size)); std::vector< homestore::dev_info > device_info; - for (auto const& path : app->devices()) { - device_info.emplace_back(std::filesystem::canonical(path).string(), homestore::HSDevType::Data); + bool has_data_dev = false; + bool has_fast_dev = false; + for (auto const& dev : app->devices()) { + auto input_dev_type = dev.type; + auto detected_type = get_device_type(dev.path.string()); + LOGD("Device {} detected as {}", dev.path.string(), detected_type); + auto final_type = (dev.type == DevType::AUTO_DETECT) ? detected_type : input_dev_type; + if (final_type == DevType::UNSUPPORTED) { + LOGW("Device {} is not supported, skipping", dev.path.string()); + continue; + } + if (input_dev_type != DevType::AUTO_DETECT && input_dev_type != final_type) { + LOGW("Device {} detected as {}, but input type is {}, using input type", dev.path.string(), detected_type, + input_dev_type); + } + auto hs_type = (final_type == DevType::HDD) ? homestore::HSDevType::Data : homestore::HSDevType::Fast; + if (hs_type == homestore::HSDevType::Data) { has_data_dev = true; } + if (hs_type == homestore::HSDevType::Fast) { has_fast_dev = true; } + device_info.emplace_back(std::filesystem::canonical(dev.path).string(), hs_type); } chunk_selector_ = std::make_shared< HeapChunkSelector >(); @@ -134,17 +158,38 @@ void HSHomeObject::init_homestore() { RELEASE_ASSERT(!_our_id.is_nil(), "Received no SvcId and need FORMAT!"); LOGW("We are starting for the first time on [{}], Formatting!!", to_string(_our_id)); - HomeStore::instance()->format_and_start({ - {HS_SERVICE::META, hs_format_params{.size_pct = 5.0}}, - {HS_SERVICE::LOG, hs_format_params{.size_pct = 10.0, .chunk_size = 32 * Mi}}, - {HS_SERVICE::REPLICATION, - hs_format_params{.size_pct = 79.0, - .num_chunks = 65000, - .block_size = _data_block_size, - .alloc_type = blk_allocator_type_t::append, - .chunk_sel_type = chunk_selector_type_t::CUSTOM}}, - {HS_SERVICE::INDEX, hs_format_params{.size_pct = 5.0}}, - }); + if (has_data_dev && has_fast_dev) { + // Hybrid mode + LOGD("Has both Data and Fast, running with Hybrid mode"); + HomeStore::instance()->format_and_start({ + {HS_SERVICE::META, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 10.0}}, + {HS_SERVICE::LOG, + hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0, .chunk_size = 32 * Mi}}, + {HS_SERVICE::INDEX, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0}}, + {HS_SERVICE::REPLICATION, + hs_format_params{.dev_type = HSDevType::Data, + .size_pct = 100.0, + .num_chunks = 65000, + .block_size = _data_block_size, + .alloc_type = blk_allocator_type_t::append, + .chunk_sel_type = chunk_selector_type_t::CUSTOM}}, + }); + } else { + auto run_on_type = has_fast_dev ? homestore::HSDevType::Fast : homestore::HSDevType::Data; + LOGD("Running with Single mode, all service on {}", run_on_type); + HomeStore::instance()->format_and_start({ + {HS_SERVICE::META, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0}}, + {HS_SERVICE::LOG, hs_format_params{.dev_type = run_on_type, .size_pct = 10.0, .chunk_size = 32 * Mi}}, + {HS_SERVICE::INDEX, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0}}, + {HS_SERVICE::REPLICATION, + hs_format_params{.dev_type = run_on_type, + .size_pct = 79.0, + .num_chunks = 65000, + .block_size = _data_block_size, + .alloc_type = blk_allocator_type_t::append, + .chunk_sel_type = chunk_selector_type_t::CUSTOM}}, + }); + } // Create a superblock that contains our SvcId auto svc_sb = homestore::superblk< svc_info_superblk_t >(_svc_meta_name); diff --git a/src/lib/homestore_backend/hs_homeobject.hpp b/src/lib/homestore_backend/hs_homeobject.hpp index faa807592..d0b9b300a 100644 --- a/src/lib/homestore_backend/hs_homeobject.hpp +++ b/src/lib/homestore_backend/hs_homeobject.hpp @@ -304,6 +304,9 @@ class HSHomeObject : public HomeObjectImpl { void persist_pg_sb(); + // helpers + DevType get_device_type(string const& devname); + public: using HomeObjectImpl::HomeObjectImpl; HSHomeObject(); diff --git a/src/lib/tests/fixture_app.hpp b/src/lib/tests/fixture_app.hpp index b1657a3d9..4078d9e90 100644 --- a/src/lib/tests/fixture_app.hpp +++ b/src/lib/tests/fixture_app.hpp @@ -32,9 +32,9 @@ class FixtureApp : public homeobject::HomeObjectApplication { if (std::filesystem::exists(path_)) std::filesystem::remove(path_); } - std::list< std::filesystem::path > devices() const override { - auto device_info = std::list< std::filesystem::path >(); - device_info.emplace_back(std::filesystem::canonical(path_)); + std::list< homeobject::device_info_t > devices() const override { + auto device_info = std::list< homeobject::device_info_t >(); + device_info.emplace_back(homeobject::device_info_t(path_)); return device_info; } diff --git a/test_package/test_package.cpp b/test_package/test_package.cpp index 2a25681e2..7a02bcac5 100644 --- a/test_package/test_package.cpp +++ b/test_package/test_package.cpp @@ -11,7 +11,9 @@ class TestApp : public homeobject::HomeObjectApplication { public: bool spdk_mode() const override { return false; } uint32_t threads() const override { return 1; } - std::list< std::filesystem::path > devices() const override { return std::list< std::filesystem::path >(); } + std::list< homeobject::device_info_t > devices() const override { + return std::list< homeobject::device_info_t >(); + } homeobject::peer_id_t discover_svcid(std::optional< homeobject::peer_id_t > const& p) const override { return boost::uuids::random_generator()(); } From fc8e161557ebfc84afdf0686749b6fc3926f4231 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Fri, 12 Apr 2024 10:49:48 -0700 Subject: [PATCH 02/11] UT with hybrid mode Signed-off-by: Xiaoxi Chen --- src/lib/homestore_backend/hs_homeobject.cpp | 4 ++-- .../homestore_backend/tests/homeobj_fixture.hpp | 2 +- src/lib/tests/fixture_app.cpp | 14 ++++++++++---- src/lib/tests/fixture_app.hpp | 16 ++++++++++++---- 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/lib/homestore_backend/hs_homeobject.cpp b/src/lib/homestore_backend/hs_homeobject.cpp index fb784f049..f30ab9c14 100644 --- a/src/lib/homestore_backend/hs_homeobject.cpp +++ b/src/lib/homestore_backend/hs_homeobject.cpp @@ -162,13 +162,13 @@ void HSHomeObject::init_homestore() { // Hybrid mode LOGD("Has both Data and Fast, running with Hybrid mode"); HomeStore::instance()->format_and_start({ - {HS_SERVICE::META, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 10.0}}, + {HS_SERVICE::META, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 9.0}}, {HS_SERVICE::LOG, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0, .chunk_size = 32 * Mi}}, {HS_SERVICE::INDEX, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0}}, {HS_SERVICE::REPLICATION, hs_format_params{.dev_type = HSDevType::Data, - .size_pct = 100.0, + .size_pct = 99.0, .num_chunks = 65000, .block_size = _data_block_size, .alloc_type = blk_allocator_type_t::append, diff --git a/src/lib/homestore_backend/tests/homeobj_fixture.hpp b/src/lib/homestore_backend/tests/homeobj_fixture.hpp index f8021d2dd..420feb71c 100644 --- a/src/lib/homestore_backend/tests/homeobj_fixture.hpp +++ b/src/lib/homestore_backend/tests/homeobj_fixture.hpp @@ -32,7 +32,7 @@ class HomeObjectFixture : public ::testing::Test { std::default_random_engine rnd_engine{rnd()}; void SetUp() override { - app = std::make_shared< FixtureApp >(); + app = std::make_shared< FixtureApp >(true /* is_hybrid */); _obj_inst = homeobject::init_homeobject(std::weak_ptr< homeobject::HomeObjectApplication >(app)); } diff --git a/src/lib/tests/fixture_app.cpp b/src/lib/tests/fixture_app.cpp index 582e145db..9e5d35c2e 100644 --- a/src/lib/tests/fixture_app.cpp +++ b/src/lib/tests/fixture_app.cpp @@ -16,11 +16,17 @@ SISL_LOGGING_INIT(HOMEOBJECT_LOG_MODS) SISL_OPTIONS_ENABLE(logging, homeobject, test_home_object) -FixtureApp::FixtureApp() { +FixtureApp::FixtureApp(bool is_hybrid) : is_hybrid_(is_hybrid) { clean(); - LOGWARN("creating device {} file with size {} ", path_, 10 * Gi); - std::ofstream ofs{path_, std::ios::binary | std::ios::out | std::ios::trunc}; - std::filesystem::resize_file(path_, 10 * Gi); + LOGWARN("creating HDD device {} file with size {} ", path_hdd_, 10 * Gi); + std::ofstream ofs{path_hdd_, std::ios::binary | std::ios::out | std::ios::trunc}; + std::filesystem::resize_file(path_hdd_, 10 * Gi); + + if (is_hybrid_) { + LOGWARN("creating SSD device {} file with size {} ", path_ssd_, 10 * Gi); + std::ofstream ofs{path_ssd_, std::ios::binary | std::ios::out | std::ios::trunc}; + std::filesystem::resize_file(path_ssd_, 10 * Gi); + } } homeobject::peer_id_t FixtureApp::discover_svcid(std::optional< homeobject::peer_id_t > const& p) const { diff --git a/src/lib/tests/fixture_app.hpp b/src/lib/tests/fixture_app.hpp index 4078d9e90..9dfcd2c99 100644 --- a/src/lib/tests/fixture_app.hpp +++ b/src/lib/tests/fixture_app.hpp @@ -19,22 +19,30 @@ using homeobject::blob_id_t; using homeobject::peer_id_t; class FixtureApp : public homeobject::HomeObjectApplication { - std::string path_{"/tmp/homeobject_test.data"}; + std::string path_hdd_{"/tmp/homeobject_test.hdd"}; + std::string path_ssd_{"/tmp/homeobject_test.ssd"}; + bool is_hybrid_{false}; public: - FixtureApp(); + FixtureApp(bool is_hybrid=false); ~FixtureApp() = default; bool spdk_mode() const override { return false; } uint32_t threads() const override { return 2; } void clean() { - if (std::filesystem::exists(path_)) std::filesystem::remove(path_); + if (std::filesystem::exists(path_hdd_)) std::filesystem::remove(path_hdd_); + if (std::filesystem::exists(path_ssd_)) std::filesystem::remove(path_ssd_); } std::list< homeobject::device_info_t > devices() const override { auto device_info = std::list< homeobject::device_info_t >(); - device_info.emplace_back(homeobject::device_info_t(path_)); + // add HDD + device_info.emplace_back(path_hdd_, homeobject::DevType::HDD); + if (is_hybrid_) { + // add SSD + device_info.emplace_back(path_ssd_, homeobject::DevType::NVME); + } return device_info; } From c68746d9ba97e35b43c8baf8c5c54c868de1d667 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Fri, 12 Apr 2024 11:19:18 -0700 Subject: [PATCH 03/11] fix a bug Signed-off-by: Xiaoxi Chen --- src/lib/homestore_backend/hs_homeobject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/homestore_backend/hs_homeobject.cpp b/src/lib/homestore_backend/hs_homeobject.cpp index f30ab9c14..a94101f25 100644 --- a/src/lib/homestore_backend/hs_homeobject.cpp +++ b/src/lib/homestore_backend/hs_homeobject.cpp @@ -133,7 +133,7 @@ void HSHomeObject::init_homestore() { LOGW("Device {} is not supported, skipping", dev.path.string()); continue; } - if (input_dev_type != DevType::AUTO_DETECT && input_dev_type != final_type) { + if (input_dev_type != DevType::AUTO_DETECT && detected_type != final_type) { LOGW("Device {} detected as {}, but input type is {}, using input type", dev.path.string(), detected_type, input_dev_type); } From f7c86205e8f24886c6165f307af5f0962621d574 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Wed, 17 Apr 2024 02:10:52 -0700 Subject: [PATCH 04/11] fix ut Signed-off-by: Xiaoxi Chen --- src/lib/homestore_backend/hs_homeobject.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/homestore_backend/hs_homeobject.cpp b/src/lib/homestore_backend/hs_homeobject.cpp index a94101f25..c5306c2f7 100644 --- a/src/lib/homestore_backend/hs_homeobject.cpp +++ b/src/lib/homestore_backend/hs_homeobject.cpp @@ -162,10 +162,10 @@ void HSHomeObject::init_homestore() { // Hybrid mode LOGD("Has both Data and Fast, running with Hybrid mode"); HomeStore::instance()->format_and_start({ - {HS_SERVICE::META, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 9.0}}, + {HS_SERVICE::META, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 9.0, .num_chunks = 64}}, {HS_SERVICE::LOG, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0, .chunk_size = 32 * Mi}}, - {HS_SERVICE::INDEX, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0}}, + {HS_SERVICE::INDEX, hs_format_params{.dev_type = HSDevType::Fast, .size_pct = 45.0, .num_chunks = 128}}, {HS_SERVICE::REPLICATION, hs_format_params{.dev_type = HSDevType::Data, .size_pct = 99.0, @@ -178,9 +178,10 @@ void HSHomeObject::init_homestore() { auto run_on_type = has_fast_dev ? homestore::HSDevType::Fast : homestore::HSDevType::Data; LOGD("Running with Single mode, all service on {}", run_on_type); HomeStore::instance()->format_and_start({ - {HS_SERVICE::META, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0}}, + // FIXME: this is to work around the issue in HS that varsize allocator doesnt work with small chunk size. + {HS_SERVICE::META, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0, .num_chunks = 1}}, {HS_SERVICE::LOG, hs_format_params{.dev_type = run_on_type, .size_pct = 10.0, .chunk_size = 32 * Mi}}, - {HS_SERVICE::INDEX, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0}}, + {HS_SERVICE::INDEX, hs_format_params{.dev_type = run_on_type, .size_pct = 5.0, .num_chunks = 1}}, {HS_SERVICE::REPLICATION, hs_format_params{.dev_type = run_on_type, .size_pct = 79.0, From 251ae9ec9782a3b52875df91905523a509766b12 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Thu, 18 Apr 2024 11:28:11 +0800 Subject: [PATCH 05/11] Sanity check for no supported device Signed-off-by: Xiaoxi Chen --- src/lib/homestore_backend/hs_homeobject.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/homestore_backend/hs_homeobject.cpp b/src/lib/homestore_backend/hs_homeobject.cpp index c5306c2f7..86c168cb6 100644 --- a/src/lib/homestore_backend/hs_homeobject.cpp +++ b/src/lib/homestore_backend/hs_homeobject.cpp @@ -142,6 +142,7 @@ void HSHomeObject::init_homestore() { if (hs_type == homestore::HSDevType::Fast) { has_fast_dev = true; } device_info.emplace_back(std::filesystem::canonical(dev.path).string(), hs_type); } + RELEASE_ASSERT(device_info.size() != 0, "No supported devices found!"); chunk_selector_ = std::make_shared< HeapChunkSelector >(); using namespace homestore; From 476fa06a4c4a44810958ca1901e3bf4c5177bdfb Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Thu, 18 Apr 2024 00:36:08 -0700 Subject: [PATCH 06/11] add operator == Signed-off-by: Xiaoxi Chen --- src/include/homeobject/homeobject.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/homeobject/homeobject.hpp b/src/include/homeobject/homeobject.hpp index 4c3d5f7d9..7d98f43be 100644 --- a/src/include/homeobject/homeobject.hpp +++ b/src/include/homeobject/homeobject.hpp @@ -16,6 +16,7 @@ class ShardManager; ENUM(DevType, uint8_t, AUTO_DETECT = 1, HDD, NVME, UNSUPPORTED); struct device_info_t { explicit device_info_t(std::string name, DevType dtype = DevType::AUTO_DETECT) : path{std::filesystem::canonical(name)}, type{dtype} {} + bool operator ==(device_info_t const& rhs) const { return path == rhs.path && type == rhs.type; } std::filesystem::path path; DevType type; }; From 00ab48e62b4e76874c7fe91c7b34017b14ce21a0 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Thu, 18 Apr 2024 09:00:51 -0700 Subject: [PATCH 07/11] depens on ^6.2 HS https://jubianchi.github.io/semver-check/#/^6.2/6.5 Caret constraint Constraint will be satisfied by versions matching >=6.2.0 <7.0.0-0. ^6.2 is a caret constraint. It means that it will match several versions. Given the constraint you entered, you will get: The next minor releases which will provide new features The next patch releases which will fix bugs ^6.2 is a range constraint. It means that it will match several versions. Signed-off-by: Xiaoxi Chen --- conanfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index 2112f7ed7..507a4b484 100644 --- a/conanfile.py +++ b/conanfile.py @@ -40,8 +40,8 @@ def build_requirements(self): self.build_requires("gtest/1.14.0") def requirements(self): - self.requires("homestore/[>=6.2, include_prerelease=True]@oss/master") - self.requires("sisl/[>=12.1, include_prerelease=True]@oss/master") + self.requires("homestore/[^6.2, include_prerelease=True]@oss/master") + self.requires("sisl/[^12.1, include_prerelease=True]@oss/master") self.requires("lz4/1.9.4", override=True) def validate(self): From ff62e8fdf910492d6ccb3aae7564425bf1a51a2b Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Thu, 18 Apr 2024 17:40:22 -0700 Subject: [PATCH 08/11] reduce #chunk on data to be 60000 that give 5536 chunks for NVME. Signed-off-by: Xiaoxi Chen --- conanfile.py | 2 +- src/lib/homestore_backend/hs_homeobject.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conanfile.py b/conanfile.py index 507a4b484..932ad5e02 100644 --- a/conanfile.py +++ b/conanfile.py @@ -40,7 +40,7 @@ def build_requirements(self): self.build_requires("gtest/1.14.0") def requirements(self): - self.requires("homestore/[^6.2, include_prerelease=True]@oss/master") + self.requires("homestore/[^6.2.0, include_prerelease=True]@oss/master") self.requires("sisl/[^12.1, include_prerelease=True]@oss/master") self.requires("lz4/1.9.4", override=True) diff --git a/src/lib/homestore_backend/hs_homeobject.cpp b/src/lib/homestore_backend/hs_homeobject.cpp index 86c168cb6..64cc2e8dc 100644 --- a/src/lib/homestore_backend/hs_homeobject.cpp +++ b/src/lib/homestore_backend/hs_homeobject.cpp @@ -170,7 +170,7 @@ void HSHomeObject::init_homestore() { {HS_SERVICE::REPLICATION, hs_format_params{.dev_type = HSDevType::Data, .size_pct = 99.0, - .num_chunks = 65000, + .num_chunks = 60000, .block_size = _data_block_size, .alloc_type = blk_allocator_type_t::append, .chunk_sel_type = chunk_selector_type_t::CUSTOM}}, @@ -186,7 +186,7 @@ void HSHomeObject::init_homestore() { {HS_SERVICE::REPLICATION, hs_format_params{.dev_type = run_on_type, .size_pct = 79.0, - .num_chunks = 65000, + .num_chunks = 60000, .block_size = _data_block_size, .alloc_type = blk_allocator_type_t::append, .chunk_sel_type = chunk_selector_type_t::CUSTOM}}, From 8436d39c032fbe40705038bb850751c224daf6ee Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Thu, 18 Apr 2024 19:36:39 -0700 Subject: [PATCH 09/11] limit HS to below 6.3 Signed-off-by: Xiaoxi Chen --- conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 932ad5e02..cedda22fa 100644 --- a/conanfile.py +++ b/conanfile.py @@ -40,7 +40,7 @@ def build_requirements(self): self.build_requires("gtest/1.14.0") def requirements(self): - self.requires("homestore/[^6.2.0, include_prerelease=True]@oss/master") + self.requires("homestore/[~6.2.0, include_prerelease=True]@oss/master") self.requires("sisl/[^12.1, include_prerelease=True]@oss/master") self.requires("lz4/1.9.4", override=True) From aa43265e5cc026a612fe9b36a42da55dcffc48f4 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Fri, 19 Apr 2024 14:20:12 +0800 Subject: [PATCH 10/11] add formattr and >> operator for SM command line parsing. cxxopts use >> to parse opts. Signed-off-by: Xiaoxi Chen --- conanfile.py | 2 +- src/include/homeobject/homeobject.hpp | 53 ++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/conanfile.py b/conanfile.py index cedda22fa..9d1ef6e7c 100644 --- a/conanfile.py +++ b/conanfile.py @@ -9,7 +9,7 @@ class HomeObjectConan(ConanFile): name = "homeobject" - version = "1.1.1" + version = "2.0.1" homepage = "https://github.com/eBay/HomeObject" description = "Blob Store built on HomeReplication" topics = ("ebay") diff --git a/src/include/homeobject/homeobject.hpp b/src/include/homeobject/homeobject.hpp index 7d98f43be..1b49125cc 100644 --- a/src/include/homeobject/homeobject.hpp +++ b/src/include/homeobject/homeobject.hpp @@ -15,8 +15,24 @@ class PGManager; class ShardManager; ENUM(DevType, uint8_t, AUTO_DETECT = 1, HDD, NVME, UNSUPPORTED); struct device_info_t { - explicit device_info_t(std::string name, DevType dtype = DevType::AUTO_DETECT) : path{std::filesystem::canonical(name)}, type{dtype} {} - bool operator ==(device_info_t const& rhs) const { return path == rhs.path && type == rhs.type; } + explicit device_info_t(std::string name, DevType dtype = DevType::AUTO_DETECT) : + path{std::filesystem::canonical(name)}, type{dtype} {} + device_info_t() = default; + bool operator==(device_info_t const& rhs) const { return path == rhs.path && type == rhs.type; } + friend std::istream& operator>>(std::istream& input, device_info_t& di) { + std::string i_path, i_type; + std::getline(input, i_path, ':'); + std::getline(input, i_type); + di.path = std::filesystem::canonical(i_path); + if (i_type == "HDD") { + di.type = DevType::HDD; + } else if (i_type == "NVME") { + di.type = DevType::NVME; + } else { + di.type = DevType::AUTO_DETECT; + } + return input; + } std::filesystem::path path; DevType type; }; @@ -60,3 +76,36 @@ class HomeObject { extern std::shared_ptr< HomeObject > init_homeobject(std::weak_ptr< HomeObjectApplication >&& application); } // namespace homeobject + // + +namespace fmt { +template <> +struct formatter< homeobject::device_info_t > { + template < typename ParseContext > + constexpr auto parse(ParseContext& ctx) { + return ctx.begin(); + } + + template < typename FormatContext > + auto format(homeobject::device_info_t const& device, FormatContext& ctx) { + std::string type; + switch (device.type) { + case homeobject::DevType::HDD: + type = "HDD"; + break; + case homeobject::DevType::NVME: + type = "NVME"; + break; + case homeobject::DevType::UNSUPPORTED: + type = "UNSUPPORTED"; + break; + case homeobject::DevType::AUTO_DETECT: + type = "AUTO_DETECT"; + break; + default: + type = "UNKNOWN"; + } + return fmt::format_to(ctx.out(), "Path: {}, Type: {}", device.path.string(), type); + } +}; +} // namespace fmt From be413477efb683faa603ff861791ac566076aa38 Mon Sep 17 00:00:00 2001 From: Xiaoxi Chen Date: Fri, 19 Apr 2024 09:43:16 -0700 Subject: [PATCH 11/11] update conan with tilde constraint Signed-off-by: Xiaoxi Chen --- conanfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conanfile.py b/conanfile.py index 9d1ef6e7c..bcdd450f0 100644 --- a/conanfile.py +++ b/conanfile.py @@ -40,8 +40,8 @@ def build_requirements(self): self.build_requires("gtest/1.14.0") def requirements(self): - self.requires("homestore/[~6.2.0, include_prerelease=True]@oss/master") - self.requires("sisl/[^12.1, include_prerelease=True]@oss/master") + self.requires("homestore/[~6.2, include_prerelease=True]@oss/master") + self.requires("sisl/[~12.1, include_prerelease=True]@oss/master") self.requires("lz4/1.9.4", override=True) def validate(self):