From 8a3edbd7275055bcadca4dddf83e348a48a47f4f Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 9 Oct 2025 14:18:54 +0800 Subject: [PATCH 1/6] tmp --- be/src/cloud/cloud_internal_service.cpp | 10 +- be/src/cloud/cloud_tablet.cpp | 51 ++++++-- be/src/cloud/cloud_tablet.h | 5 +- be/src/cloud/cloud_warm_up_manager.cpp | 15 +-- be/src/cloud/cloud_warm_up_manager.h | 1 + be/test/cloud/cloud_tablet_test.cpp | 149 ++++++++++++++---------- 6 files changed, 146 insertions(+), 85 deletions(-) diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 07880da083d22e..b4496110c57a4e 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -228,7 +228,7 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c expiration_time = 0; } - if (!tablet->add_rowset_warmup_state(rs_meta, WarmUpState::TRIGGERED_BY_JOB)) { + if (!tablet->add_rowset_warmup_state(rs_meta, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)) { LOG(INFO) << "found duplicate warmup task for rowset " << rowset_id.to_string() << ", skip it"; continue; @@ -279,8 +279,9 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c LOG(WARNING) << "download segment failed, tablet_id: " << tablet_id << " rowset_id: " << rowset_id.to_string() << ", error: " << st; } - if (tablet->complete_rowset_segment_warmup(rowset_id, st, 1, 0) == - WarmUpState::DONE) { + if (tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + rowset_id, st, 1, + 0) == WarmUpState::DONE) { VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" << rowset_id.to_string() << ") completed"; } @@ -352,7 +353,8 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c LOG(WARNING) << "download inverted index failed, tablet_id: " << tablet_id << " rowset_id: " << rowset_id << ", error: " << st; } - if (tablet->complete_rowset_segment_warmup(rowset_id, st, 0, 1) == + if (tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset_id, st, 0, 1) == WarmUpState::DONE) { VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" << rowset_id.to_string() << ") completed"; diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index f1f02ecc38dcd0..878d7b27bdf528 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -128,6 +128,8 @@ bvar::Adder g_file_cache_warm_up_rowset_triggered_by_job_num( "file_cache_warm_up_rowset_triggered_by_job_num"); bvar::Adder g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num( "file_cache_warm_up_rowset_triggered_by_sync_rowset_num"); +bvar::Adder g_file_cache_warm_up_rowset_triggered_by_event_driven_num( + "file_cache_warm_up_rowset_triggered_by_event_driven_num"); bvar::LatencyRecorder g_file_cache_warm_up_rowset_all_segments_latency( "file_cache_warm_up_rowset_all_segments_latency"); @@ -476,7 +478,7 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ std::chrono::seconds(sleep_time)); } }); - self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st, 1, 0); + self->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset_meta->rowset_id(), st, 1, 0); if (!st) { LOG_WARNING("add rowset warm up error ").error(st); } @@ -508,7 +510,7 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ std::chrono::seconds(sleep_time)); // clang-format off }); - self->complete_rowset_segment_warmup(rowset_meta->rowset_id(), st, 0, 1); + self->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset_meta->rowset_id(), st, 0, 1); if (!st) { LOG_WARNING("add rowset warm up error ").error(st); } @@ -1673,26 +1675,52 @@ void CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, WarmUpState state, std::chrono::steady_clock::time_point start_tp) { - if (_rowset_warm_up_states.contains(rowset.rowset_id())) { - return false; + auto rowset_id = rowset.rowset_id(); + + // Check if rowset already has warmup state + if (_rowset_warm_up_states.contains(rowset_id)) { + auto existing_state = _rowset_warm_up_states[rowset_id].state; + + // For job-triggered warmup (one-time and periodic warmup), allow it to proceed + // except when there's already another job-triggered warmup in progress + if (state == WarmUpState::TRIGGERED_BY_JOB) { + if (existing_state == WarmUpState::TRIGGERED_BY_JOB) { + // Same job type already in progress, skip to avoid duplicate warmup + return false; + } + // Override other states (EVENT_DRIVEN, SYNC_ROWSET) but not another JOB + } else { + // For non-job warmup (EVENT_DRIVEN, SYNC_ROWSET), skip if any warmup exists + return false; + } } + if (state == WarmUpState::TRIGGERED_BY_JOB) { g_file_cache_warm_up_rowset_triggered_by_job_num << 1; } else if (state == WarmUpState::TRIGGERED_BY_SYNC_ROWSET) { g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num << 1; + } else if (state == WarmUpState::TRIGGERED_BY_EVENT_DRIVEN) { + g_file_cache_warm_up_rowset_triggered_by_event_driven_num << 1; } - _rowset_warm_up_states[rowset.rowset_id()] = { + _rowset_warm_up_states[rowset_id] = { .state = state, .num_segments = rowset.num_segments(), .start_tp = start_tp}; return true; } -WarmUpState CloudTablet::complete_rowset_segment_warmup(RowsetId rowset_id, Status status, +WarmUpState CloudTablet::complete_rowset_segment_warmup(WarmUpState trigger_source, + RowsetId rowset_id, Status status, int64_t segment_num, int64_t inverted_idx_num) { std::lock_guard wlock(_meta_lock); - if (!_rowset_warm_up_states.contains(rowset_id)) { + auto it = _rowset_warm_up_states.find(rowset_id); + if (it == _rowset_warm_up_states.end()) { return WarmUpState::NONE; } + auto& warmup_info = it->second; + if (warmup_info.state != trigger_source) { + // Only the same trigger source can update the state + return warmup_info.state; + } VLOG_DEBUG << "complete rowset segment warmup for rowset " << rowset_id << ", " << status; if (segment_num > 0) { g_file_cache_warm_up_segment_complete_num << segment_num; @@ -1706,17 +1734,16 @@ WarmUpState CloudTablet::complete_rowset_segment_warmup(RowsetId rowset_id, Stat g_file_cache_warm_up_inverted_idx_failed_num << inverted_idx_num; } } - _rowset_warm_up_states[rowset_id].done(segment_num, inverted_idx_num); + warmup_info.done(segment_num, inverted_idx_num); if (_rowset_warm_up_states[rowset_id].has_finished()) { g_file_cache_warm_up_rowset_complete_num << 1; auto cost = std::chrono::duration_cast( - std::chrono::steady_clock::now() - - _rowset_warm_up_states[rowset_id].start_tp) + std::chrono::steady_clock::now() - warmup_info.start_tp) .count(); g_file_cache_warm_up_rowset_all_segments_latency << cost; - _rowset_warm_up_states[rowset_id].state = WarmUpState::DONE; + warmup_info.state = WarmUpState::DONE; } - return _rowset_warm_up_states[rowset_id].state; + return warmup_info.state; } bool CloudTablet::is_rowset_warmed_up(const RowsetId& rowset_id) const { diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 425196301ec1af..9c89b124280370 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -330,8 +330,9 @@ class CloudTablet final : public BaseTablet { std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now()); void update_rowset_warmup_state_inverted_idx_num(RowsetId rowset_id, int64_t delta); void update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId rowset_id, int64_t delta); - WarmUpState complete_rowset_segment_warmup(RowsetId rowset_id, Status status, - int64_t segment_num, int64_t inverted_idx_num); + WarmUpState complete_rowset_segment_warmup(WarmUpState trigger_source, RowsetId rowset_id, + Status status, int64_t segment_num, + int64_t inverted_idx_num); bool is_rowset_warmed_up(const RowsetId& rowset_id) const; diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index fe61facd222e45..9be31317a77ff1 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -249,7 +249,8 @@ void CloudWarmUpManager::handle_jobs() { VLOG_DEBUG << "warmup rowset " << rs->version() << " segment " << seg_id << " completed"; if (tablet->complete_rowset_segment_warmup( - rs->rowset_id(), st, 1, 0) == WarmUpState::DONE) { + WarmUpState::TRIGGERED_BY_JOB, rs->rowset_id(), st, 1, + 0) == WarmUpState::DONE) { VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; } }); @@ -290,9 +291,9 @@ void CloudWarmUpManager::handle_jobs() { VLOG_DEBUG << "warmup rowset " << rs->version() << " segment " << seg_id << "inverted idx:" << idx_path << " completed"; - if (tablet->complete_rowset_segment_warmup(rs->rowset_id(), - st, 0, 1) == - WarmUpState::DONE) { + if (tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_JOB, rs->rowset_id(), + st, 0, 1) == WarmUpState::DONE) { VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; } @@ -311,9 +312,9 @@ void CloudWarmUpManager::handle_jobs() { VLOG_DEBUG << "warmup rowset " << rs->version() << " segment " << seg_id << "inverted idx:" << idx_path << " completed"; - if (tablet->complete_rowset_segment_warmup(rs->rowset_id(), - st, 0, 1) == - WarmUpState::DONE) { + if (tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_JOB, rs->rowset_id(), + st, 0, 1) == WarmUpState::DONE) { VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; } diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index 8eb42a5d1cc733..39ea541f08f6ab 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -42,6 +42,7 @@ enum class DownloadType { enum class WarmUpState : int { NONE, TRIGGERED_BY_SYNC_ROWSET, + TRIGGERED_BY_EVENT_DRIVEN, TRIGGERED_BY_JOB, DONE, }; diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index fe9751ff7bfbc9..4d10b896313c62 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -83,18 +83,18 @@ TEST_F(CloudTabletWarmUpStateTest, TestGetRowsetWarmupStateNonExistent) { EXPECT_EQ(state, WarmUpState::NONE); } -// Test add_rowset_warmup_state with TRIGGERED_BY_JOB state +// Test add_rowset_warmup_state with TRIGGERED_BY_EVENT_DRIVEN state TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredByJob) { auto rowset = create_rowset(Version(1, 1), 5); ASSERT_NE(rowset, nullptr); bool result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_TRUE(result); // Verify the state is correctly set WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); } // Test add_rowset_warmup_state with TRIGGERED_BY_SYNC_ROWSET state @@ -118,7 +118,7 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddDuplicateRowsetWarmupState) { // First addition should succeed bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_TRUE(result1); // Second addition should fail @@ -128,15 +128,15 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddDuplicateRowsetWarmupState) { // State should remain the original one WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); } // Test complete_rowset_segment_warmup for non-existent rowset TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupNonExistent) { auto non_existent_id = _engine.next_rowset_id(); - WarmUpState result = - _tablet->complete_rowset_segment_warmup(non_existent_id, Status::OK(), 1, 0); + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_SYNC_ROWSET, non_existent_id, Status::OK(), 1, 0); EXPECT_EQ(result, WarmUpState::NONE); } @@ -147,22 +147,22 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupPartial) { // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_TRUE(add_result); - // Complete one segment, should still be in TRIGGERED_BY_JOB state - WarmUpState result1 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB); + // Complete one segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state + WarmUpState result1 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); - // Complete second segment, should still be in TRIGGERED_BY_JOB state - WarmUpState result2 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_JOB); + // Complete second segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state + WarmUpState result2 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); - // Verify current state is still TRIGGERED_BY_JOB + // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); } // Test complete_rowset_segment_warmup with full completion @@ -176,13 +176,13 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupFull) { EXPECT_TRUE(add_result); // Complete first segment - WarmUpState result1 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); + WarmUpState result1 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_SYNC_ROWSET); // Complete second segment, should transition to DONE state - WarmUpState result2 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); + WarmUpState result2 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); EXPECT_EQ(result2, WarmUpState::DONE); // Verify final state is DONE @@ -197,25 +197,25 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_TRUE(add_result); _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); // Complete one segment file - WarmUpState result1 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB); + WarmUpState result1 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); - // Complete inverted index file, should still be in TRIGGERED_BY_JOB state - WarmUpState result2 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 0, 1); - EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_JOB); + // Complete inverted index file, should still be in TRIGGERED_BY_EVENT_DRIVEN state + WarmUpState result2 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); + EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); - // Verify current state is still TRIGGERED_BY_JOB + // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); } // Test complete_rowset_segment_warmup with inverted index file, full completion @@ -225,19 +225,19 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_TRUE(add_result); _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); // Complete segment file - WarmUpState result1 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_JOB); + WarmUpState result1 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); // Complete inverted index file - WarmUpState result2 = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 0, 1); + WarmUpState result2 = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); EXPECT_EQ(result2, WarmUpState::DONE); // Verify final state is DONE @@ -252,13 +252,13 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithError) { // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_TRUE(add_result); // Complete with error status, should still transition to DONE when all segments complete Status error_status = Status::InternalError("Test error"); - WarmUpState result = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), error_status, 1, 0); + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), error_status, 1, 0); EXPECT_EQ(result, WarmUpState::DONE); // Verify final state is DONE even with error @@ -277,28 +277,31 @@ TEST_F(CloudTabletWarmUpStateTest, TestMultipleRowsetsWarmupState) { // Add multiple rowsets EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB)); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)); EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()), WarmUpState::TRIGGERED_BY_SYNC_ROWSET)); EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset3->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB)); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)); // Verify all states EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), WarmUpState::TRIGGERED_BY_SYNC_ROWSET); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); // Complete rowset1 (2 segments) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState::TRIGGERED_BY_JOB); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0), + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0), WarmUpState::DONE); // Complete rowset3 (1 segment) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset3->rowset_id(), Status::OK(), 1, 0), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + rowset3->rowset_id(), Status::OK(), 1, 0), WarmUpState::DONE); // Verify states after completion @@ -315,16 +318,16 @@ TEST_F(CloudTabletWarmUpStateTest, TestWarmupStateWithZeroSegments) { // Add rowset with zero segments bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_TRUE(add_result); // State should be immediately ready for completion since there are no segments to warm up WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); // Any completion call should handle the edge case gracefully - WarmUpState result = - _tablet->complete_rowset_segment_warmup(rowset->rowset_id(), Status::OK(), 1, 0); + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); // With 0 segments, the counter should already be 0, so this should transition to DONE EXPECT_EQ(result, WarmUpState::DONE); } @@ -338,22 +341,48 @@ TEST_F(CloudTabletWarmUpStateTest, TestConcurrentWarmupStateAccess) { // Add rowsets from different "threads" (simulated by sequential calls) EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB)); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)); EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()), WarmUpState::TRIGGERED_BY_SYNC_ROWSET)); // Interleaved completion operations - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState::TRIGGERED_BY_JOB); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset2->rowset_id(), Status::OK(), 1, 0), + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0), + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_SYNC_ROWSET, + rowset2->rowset_id(), Status::OK(), 1, 0), WarmUpState::TRIGGERED_BY_SYNC_ROWSET); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0), + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); // Check states are maintained correctly EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), - WarmUpState::TRIGGERED_BY_JOB); + WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), WarmUpState::TRIGGERED_BY_SYNC_ROWSET); } + +// Test only when the trigger source matches can the state be updated +TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupTriggerSource) { + auto rowset = create_rowset(Version(13, 13), 1); + ASSERT_NE(rowset, nullptr); + // Add rowset warmup state + bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpState::TRIGGERED_BY_JOB); + EXPECT_TRUE(add_result); + + // Attempt to complete with a different trigger source, should not update state + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result, WarmUpState::TRIGGERED_BY_JOB); + result = _tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result, WarmUpState::TRIGGERED_BY_JOB); + + // Now complete with the correct trigger source + result = _tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_JOB, + rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result, WarmUpState::DONE); +} } // namespace doris From 72a1afb07f501d606f6af7b694b96c0648102c8d Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 9 Oct 2025 15:26:25 +0800 Subject: [PATCH 2/6] tmp --- be/src/cloud/cloud_internal_service.cpp | 18 +-- be/src/cloud/cloud_tablet.cpp | 89 +++++++------ be/src/cloud/cloud_tablet.h | 33 +++-- be/src/cloud/cloud_warm_up_manager.cpp | 26 ++-- be/src/cloud/cloud_warm_up_manager.h | 8 -- be/test/cloud/cloud_tablet_test.cpp | 164 ++++++++++++------------ 6 files changed, 184 insertions(+), 154 deletions(-) diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index b4496110c57a4e..24f60ff1eb59cb 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -20,6 +20,7 @@ #include #include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" #include "cloud/cloud_warm_up_manager.h" #include "cloud/config.h" @@ -228,7 +229,7 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c expiration_time = 0; } - if (!tablet->add_rowset_warmup_state(rs_meta, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)) { + if (!tablet->add_rowset_warmup_state(rs_meta, WarmUpTriggerSource::EVENT_DRIVEN)) { LOG(INFO) << "found duplicate warmup task for rowset " << rowset_id.to_string() << ", skip it"; continue; @@ -279,9 +280,9 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c LOG(WARNING) << "download segment failed, tablet_id: " << tablet_id << " rowset_id: " << rowset_id.to_string() << ", error: " << st; } - if (tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, - rowset_id, st, 1, - 0) == WarmUpState::DONE) { + if (tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, + rowset_id, st, 1, 0) + .trigger_source == WarmUpTriggerSource::EVENT_DRIVEN) { VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" << rowset_id.to_string() << ") completed"; } @@ -353,9 +354,9 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c LOG(WARNING) << "download inverted index failed, tablet_id: " << tablet_id << " rowset_id: " << rowset_id << ", error: " << st; } - if (tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset_id, st, 0, 1) == - WarmUpState::DONE) { + if (tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, + rowset_id, st, 0, 1) + .trigger_source == WarmUpTriggerSource::EVENT_DRIVEN) { VLOG_DEBUG << "warmup rowset " << version.to_string() << "(" << rowset_id.to_string() << ") completed"; } @@ -375,7 +376,8 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c }; g_file_cache_event_driven_warm_up_submitted_index_num << 1; g_file_cache_event_driven_warm_up_submitted_index_size << idx_size; - tablet->update_rowset_warmup_state_inverted_idx_num(rowset_id, 1); + tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::EVENT_DRIVEN, rowset_id, 1); if (wait) { wait->add_count(); } diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 878d7b27bdf528..0bbb2dc1e98a56 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -445,8 +445,8 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ if (!warm_up_state_updated) { VLOG_DEBUG << "warm up rowset " << rs->version() << "(" << rs->rowset_id() << ") triggerd by sync rowset"; - if (!add_rowset_warmup_state_unlocked( - *(rs->rowset_meta()), WarmUpState::TRIGGERED_BY_SYNC_ROWSET)) { + if (!add_rowset_warmup_state_unlocked(*(rs->rowset_meta()), + WarmUpTriggerSource::SYNC_ROWSET)) { LOG(INFO) << "found duplicate warmup task for rowset " << rs->rowset_id() << ", skip it"; break; @@ -478,7 +478,7 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ std::chrono::seconds(sleep_time)); } }); - self->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset_meta->rowset_id(), st, 1, 0); + self->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET, rowset_meta->rowset_id(), st, 1, 0); if (!st) { LOG_WARNING("add rowset warm up error ").error(st); } @@ -510,13 +510,13 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ std::chrono::seconds(sleep_time)); // clang-format off }); - self->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset_meta->rowset_id(), st, 0, 1); + self->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET, rowset_meta->rowset_id(), st, 0, 1); if (!st) { LOG_WARNING("add rowset warm up error ").error(st); } }}, }; - self->update_rowset_warmup_state_inverted_idx_num_unlocked(rowset_meta->rowset_id(), 1); + self->update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource::SYNC_ROWSET, rowset_meta->rowset_id(), 1); _engine.file_cache_block_downloader().submit_download_task(std::move(meta)); g_file_cache_cloud_tablet_submitted_index_num << 1; g_file_cache_cloud_tablet_submitted_index_size << idx_size; @@ -1649,31 +1649,41 @@ Status CloudTablet::check_delete_bitmap_cache(int64_t txn_id, WarmUpState CloudTablet::get_rowset_warmup_state(RowsetId rowset_id) { std::shared_lock rlock(_meta_lock); if (!_rowset_warm_up_states.contains(rowset_id)) { - return WarmUpState::NONE; + return {.trigger_source = WarmUpTriggerSource::NONE, .progress = WarmUpProgress::NONE}; } - return _rowset_warm_up_states[rowset_id].state; + auto& warmup_info = _rowset_warm_up_states[rowset_id]; + warmup_info.update_state(); + return warmup_info.state; } -bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpState state, +bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpTriggerSource source, std::chrono::steady_clock::time_point start_tp) { std::lock_guard wlock(_meta_lock); - return add_rowset_warmup_state_unlocked(rowset, state, start_tp); + return add_rowset_warmup_state_unlocked(rowset, source, start_tp); } -void CloudTablet::update_rowset_warmup_state_inverted_idx_num(RowsetId rowset_id, int64_t delta) { +void CloudTablet::update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource source, + RowsetId rowset_id, int64_t delta) { std::lock_guard wlock(_meta_lock); - update_rowset_warmup_state_inverted_idx_num_unlocked(rowset_id, delta); + update_rowset_warmup_state_inverted_idx_num_unlocked(source, rowset_id, delta); } -void CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId rowset_id, +void CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source, + RowsetId rowset_id, int64_t delta) { - if (!_rowset_warm_up_states.contains(rowset_id)) { + auto it = _rowset_warm_up_states.find(rowset_id); + if (it == _rowset_warm_up_states.end()) { + return; + } + if (it->second.state.trigger_source != source) { + // Only the same trigger source can update the state return; } - _rowset_warm_up_states[rowset_id].num_inverted_idx += delta; + it->second.num_inverted_idx += delta; } -bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, WarmUpState state, +bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, + WarmUpTriggerSource source, std::chrono::steady_clock::time_point start_tp) { auto rowset_id = rowset.rowset_id(); @@ -1683,8 +1693,8 @@ bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, War // For job-triggered warmup (one-time and periodic warmup), allow it to proceed // except when there's already another job-triggered warmup in progress - if (state == WarmUpState::TRIGGERED_BY_JOB) { - if (existing_state == WarmUpState::TRIGGERED_BY_JOB) { + if (source == WarmUpTriggerSource::JOB) { + if (existing_state.trigger_source == WarmUpTriggerSource::JOB) { // Same job type already in progress, skip to avoid duplicate warmup return false; } @@ -1695,29 +1705,42 @@ bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, War } } - if (state == WarmUpState::TRIGGERED_BY_JOB) { + if (source == WarmUpTriggerSource::JOB) { g_file_cache_warm_up_rowset_triggered_by_job_num << 1; - } else if (state == WarmUpState::TRIGGERED_BY_SYNC_ROWSET) { + } else if (source == WarmUpTriggerSource::SYNC_ROWSET) { g_file_cache_warm_up_rowset_triggered_by_sync_rowset_num << 1; - } else if (state == WarmUpState::TRIGGERED_BY_EVENT_DRIVEN) { + } else if (source == WarmUpTriggerSource::EVENT_DRIVEN) { g_file_cache_warm_up_rowset_triggered_by_event_driven_num << 1; } _rowset_warm_up_states[rowset_id] = { - .state = state, .num_segments = rowset.num_segments(), .start_tp = start_tp}; + .state = {.trigger_source = source, .progress = WarmUpProgress::DOING}, + .num_segments = rowset.num_segments(), + .start_tp = start_tp}; return true; } -WarmUpState CloudTablet::complete_rowset_segment_warmup(WarmUpState trigger_source, +void CloudTablet::RowsetWarmUpInfo::update_state() { + if (has_finished()) { + g_file_cache_warm_up_rowset_complete_num << 1; + auto cost = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_tp) + .count(); + g_file_cache_warm_up_rowset_all_segments_latency << cost; + state.progress = WarmUpProgress::DONE; + } +} + +WarmUpState CloudTablet::complete_rowset_segment_warmup(WarmUpTriggerSource trigger_source, RowsetId rowset_id, Status status, int64_t segment_num, int64_t inverted_idx_num) { std::lock_guard wlock(_meta_lock); auto it = _rowset_warm_up_states.find(rowset_id); if (it == _rowset_warm_up_states.end()) { - return WarmUpState::NONE; + return {.trigger_source = WarmUpTriggerSource::NONE, .progress = WarmUpProgress::NONE}; } auto& warmup_info = it->second; - if (warmup_info.state != trigger_source) { + if (warmup_info.state.trigger_source != trigger_source) { // Only the same trigger source can update the state return warmup_info.state; } @@ -1735,14 +1758,6 @@ WarmUpState CloudTablet::complete_rowset_segment_warmup(WarmUpState trigger_sour } } warmup_info.done(segment_num, inverted_idx_num); - if (_rowset_warm_up_states[rowset_id].has_finished()) { - g_file_cache_warm_up_rowset_complete_num << 1; - auto cost = std::chrono::duration_cast( - std::chrono::steady_clock::now() - warmup_info.start_tp) - .count(); - g_file_cache_warm_up_rowset_all_segments_latency << cost; - warmup_info.state = WarmUpState::DONE; - } return warmup_info.state; } @@ -1751,13 +1766,15 @@ bool CloudTablet::is_rowset_warmed_up(const RowsetId& rowset_id) const { if (it == _rowset_warm_up_states.end()) { return false; } - return it->second.state == WarmUpState::DONE; + return it->second.state.progress == WarmUpProgress::DONE; } void CloudTablet::add_warmed_up_rowset(const RowsetId& rowset_id) { - _rowset_warm_up_states[rowset_id] = {.state = WarmUpState::DONE, - .num_segments = 1, - .start_tp = std::chrono::steady_clock::now()}; + _rowset_warm_up_states[rowset_id] = { + .state = {.trigger_source = WarmUpTriggerSource::SYNC_ROWSET, + .progress = WarmUpProgress::DONE}, + .num_segments = 1, + .start_tp = std::chrono::steady_clock::now()}; } #include "common/compile_check_end.h" diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 9c89b124280370..5eb0e7830e868d 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -26,7 +26,15 @@ namespace doris { class CloudStorageEngine; -enum class WarmUpState : int; + +enum class WarmUpTriggerSource : int { NONE, SYNC_ROWSET, EVENT_DRIVEN, JOB }; + +enum class WarmUpProgress : int { NONE, DOING, DONE }; + +struct WarmUpState { + WarmUpTriggerSource trigger_source {WarmUpTriggerSource::NONE}; + WarmUpProgress progress {WarmUpProgress::NONE}; +}; struct SyncRowsetStats { int64_t get_remote_rowsets_num {0}; @@ -326,13 +334,15 @@ class CloudTablet final : public BaseTablet { // Add warmup state management WarmUpState get_rowset_warmup_state(RowsetId rowset_id); bool add_rowset_warmup_state( - const RowsetMeta& rowset, WarmUpState state, + const RowsetMeta& rowset, WarmUpTriggerSource source, std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now()); - void update_rowset_warmup_state_inverted_idx_num(RowsetId rowset_id, int64_t delta); - void update_rowset_warmup_state_inverted_idx_num_unlocked(RowsetId rowset_id, int64_t delta); - WarmUpState complete_rowset_segment_warmup(WarmUpState trigger_source, RowsetId rowset_id, - Status status, int64_t segment_num, - int64_t inverted_idx_num); + void update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource source, RowsetId rowset_id, + int64_t delta); + void update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source, + RowsetId rowset_id, int64_t delta); + WarmUpState complete_rowset_segment_warmup(WarmUpTriggerSource trigger_source, + RowsetId rowset_id, Status status, + int64_t segment_num, int64_t inverted_idx_num); bool is_rowset_warmed_up(const RowsetId& rowset_id) const; @@ -344,8 +354,8 @@ class CloudTablet final : public BaseTablet { auto tmp = fmt::format("{}{}", rs->rowset_id().to_string(), rs->version().to_string()); if (_rowset_warm_up_states.contains(rs->rowset_id())) { tmp += fmt::format( - ", state={}, segments_warmed_up={}/{}, inverted_idx_warmed_up={}/{}", - _rowset_warm_up_states.at(rs->rowset_id()).state, + ", progress={}, segments_warmed_up={}/{}, inverted_idx_warmed_up={}/{}", + _rowset_warm_up_states.at(rs->rowset_id()).state.progress, _rowset_warm_up_states.at(rs->rowset_id()).num_segments_warmed_up, _rowset_warm_up_states.at(rs->rowset_id()).num_segments, _rowset_warm_up_states.at(rs->rowset_id()).num_inverted_idx_warmed_up, @@ -364,7 +374,7 @@ class CloudTablet final : public BaseTablet { Status sync_if_not_running(SyncRowsetStats* stats = nullptr); bool add_rowset_warmup_state_unlocked( - const RowsetMeta& rowset, WarmUpState state, + const RowsetMeta& rowset, WarmUpTriggerSource source, std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now()); // used by capture_rs_reader_xxx functions @@ -439,12 +449,15 @@ class CloudTablet final : public BaseTablet { void done(int64_t num_segments, int64_t num_inverted_idx) { num_segments_warmed_up += num_segments; num_inverted_idx_warmed_up += num_inverted_idx; + update_state(); } bool has_finished() const { return (num_segments_warmed_up >= num_segments) && (num_inverted_idx_warmed_up >= num_inverted_idx); } + + void update_state(); }; std::unordered_map _rowset_warm_up_states; diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 9be31317a77ff1..648318c039098d 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -27,6 +27,7 @@ #include #include "bvar/bvar.h" +#include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" #include "cloud/config.h" #include "common/cast_set.h" @@ -234,7 +235,7 @@ void CloudWarmUpManager::handle_jobs() { if (expiration_time <= UnixSeconds()) { expiration_time = 0; } - if (!tablet->add_rowset_warmup_state(*rs, WarmUpState::TRIGGERED_BY_JOB)) { + if (!tablet->add_rowset_warmup_state(*rs, WarmUpTriggerSource::JOB)) { LOG(INFO) << "found duplicate warmup task for rowset " << rs->rowset_id() << ", skip it"; continue; @@ -248,9 +249,10 @@ void CloudWarmUpManager::handle_jobs() { [tablet, rs, seg_id](Status st) { VLOG_DEBUG << "warmup rowset " << rs->version() << " segment " << seg_id << " completed"; - if (tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_JOB, rs->rowset_id(), st, 1, - 0) == WarmUpState::DONE) { + if (tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::JOB, + rs->rowset_id(), st, 1, + 0) + .trigger_source == WarmUpTriggerSource::JOB) { VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; } }); @@ -284,7 +286,8 @@ void CloudWarmUpManager::handle_jobs() { } } } - tablet->update_rowset_warmup_state_inverted_idx_num(rs->rowset_id(), 1); + tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::JOB, rs->rowset_id(), 1); submit_download_tasks( idx_path, file_size, storage_resource.value()->fs, expiration_time, wait, true, [=](Status st) { @@ -292,8 +295,9 @@ void CloudWarmUpManager::handle_jobs() { << " segment " << seg_id << "inverted idx:" << idx_path << " completed"; if (tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_JOB, rs->rowset_id(), - st, 0, 1) == WarmUpState::DONE) { + WarmUpTriggerSource::JOB, rs->rowset_id(), + st, 0, 1) + .trigger_source == WarmUpTriggerSource::JOB) { VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; } @@ -305,7 +309,8 @@ void CloudWarmUpManager::handle_jobs() { storage_resource.value()->remote_idx_v2_path(*rs, seg_id); file_size = idx_file_info.has_index_size() ? idx_file_info.index_size() : -1; - tablet->update_rowset_warmup_state_inverted_idx_num(rs->rowset_id(), 1); + tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::JOB, rs->rowset_id(), 1); submit_download_tasks( idx_path, file_size, storage_resource.value()->fs, expiration_time, wait, true, [=](Status st) { @@ -313,8 +318,9 @@ void CloudWarmUpManager::handle_jobs() { << " segment " << seg_id << "inverted idx:" << idx_path << " completed"; if (tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_JOB, rs->rowset_id(), - st, 0, 1) == WarmUpState::DONE) { + WarmUpTriggerSource::JOB, rs->rowset_id(), + st, 0, 1) + .trigger_source == WarmUpTriggerSource::JOB) { VLOG_DEBUG << "warmup rowset " << rs->version() << " completed"; } diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index 39ea541f08f6ab..dfa4f6e2be688a 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -39,14 +39,6 @@ enum class DownloadType { S3, }; -enum class WarmUpState : int { - NONE, - TRIGGERED_BY_SYNC_ROWSET, - TRIGGERED_BY_EVENT_DRIVEN, - TRIGGERED_BY_JOB, - DONE, -}; - struct JobMeta { JobMeta() = default; JobMeta(const TJobMeta& meta); diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index 4d10b896313c62..ff1fbf3cc833cf 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -80,7 +80,7 @@ TEST_F(CloudTabletWarmUpStateTest, TestGetRowsetWarmupStateNonExistent) { auto non_existent_id = _engine.next_rowset_id(); WarmUpState state = _tablet->get_rowset_warmup_state(non_existent_id); - EXPECT_EQ(state, WarmUpState::NONE); + EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::NONE, WarmUpProgress::NONE}); } // Test add_rowset_warmup_state with TRIGGERED_BY_EVENT_DRIVEN state @@ -89,12 +89,12 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredByJob) { ASSERT_NE(rowset, nullptr); bool result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(result); // Verify the state is correctly set WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); } // Test add_rowset_warmup_state with TRIGGERED_BY_SYNC_ROWSET state @@ -103,12 +103,12 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredBySyncRowset ASSERT_NE(rowset, nullptr); bool result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + WarmUpTriggerSource::SYNC_ROWSET); EXPECT_TRUE(result); // Verify the state is correctly set WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); } // Test adding duplicate rowset warmup state should fail @@ -118,17 +118,17 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddDuplicateRowsetWarmupState) { // First addition should succeed bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(result1); // Second addition should fail bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + WarmUpTriggerSource::SYNC_ROWSET); EXPECT_FALSE(result2); // State should remain the original one WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); } // Test complete_rowset_segment_warmup for non-existent rowset @@ -136,8 +136,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupNonExistent) { auto non_existent_id = _engine.next_rowset_id(); WarmUpState result = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_SYNC_ROWSET, non_existent_id, Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpState::NONE); + WarmUpTriggerSource::SYNC_ROWSET, non_existent_id, Status::OK(), 1, 0); + EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::NONE, WarmUpProgress::NONE}); } // Test complete_rowset_segment_warmup with partial completion @@ -147,22 +147,23 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupPartial) { // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(add_result); // Complete one segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state WarmUpState result1 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); // Complete second segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state WarmUpState result2 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + EXPECT_EQ(current_state, + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); } // Test complete_rowset_segment_warmup with full completion @@ -172,22 +173,22 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupFull) { // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + WarmUpTriggerSource::SYNC_ROWSET); EXPECT_TRUE(add_result); // Complete first segment WarmUpState result1 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); // Complete second segment, should transition to DONE state WarmUpState result2 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result2, WarmUpState::DONE); + WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DONE}); // Verify final state is DONE WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(final_state, WarmUpState::DONE); + EXPECT_EQ(final_state, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DONE}); } // Test complete_rowset_segment_warmup with inverted index file, partial completion @@ -197,25 +198,28 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(add_result); - _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); - _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); + _tablet->update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource::EVENT_DRIVEN, + rowset->rowset_id(), 1); + _tablet->update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource::EVENT_DRIVEN, + rowset->rowset_id(), 1); // Complete one segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); // Complete inverted index file, should still be in TRIGGERED_BY_EVENT_DRIVEN state WarmUpState result2 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); - EXPECT_EQ(result2, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); + EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(current_state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + EXPECT_EQ(current_state, + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); } // Test complete_rowset_segment_warmup with inverted index file, full completion @@ -225,24 +229,25 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(add_result); - _tablet->update_rowset_warmup_state_inverted_idx_num(rowset->rowset_id(), 1); + _tablet->update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource::EVENT_DRIVEN, + rowset->rowset_id(), 1); // Complete segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); // Complete inverted index file WarmUpState result2 = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); - EXPECT_EQ(result2, WarmUpState::DONE); + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); + EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); // Verify final state is DONE WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(final_state, WarmUpState::DONE); + EXPECT_EQ(final_state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); } // Test complete_rowset_segment_warmup with error status @@ -252,18 +257,18 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithError) { // Add rowset warmup state bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(add_result); // Complete with error status, should still transition to DONE when all segments complete Status error_status = Status::InternalError("Test error"); WarmUpState result = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), error_status, 1, 0); - EXPECT_EQ(result, WarmUpState::DONE); + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), error_status, 1, 0); + EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); // Verify final state is DONE even with error WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(final_state, WarmUpState::DONE); + EXPECT_EQ(final_state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); } // Test multiple rowsets warmup state management @@ -277,37 +282,38 @@ TEST_F(CloudTabletWarmUpStateTest, TestMultipleRowsetsWarmupState) { // Add multiple rowsets EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)); + WarmUpTriggerSource::EVENT_DRIVEN)); EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET)); + WarmUpTriggerSource::SYNC_ROWSET)); EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset3->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)); + WarmUpTriggerSource::EVENT_DRIVEN)); // Verify all states EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); // Complete rowset1 (2 segments) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState::DONE); + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); // Complete rowset3 (1 segment) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, rowset3->rowset_id(), Status::OK(), 1, 0), - WarmUpState::DONE); + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); // Verify states after completion - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), WarmUpState::DONE); + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), WarmUpState::DONE); } @@ -318,18 +324,12 @@ TEST_F(CloudTabletWarmUpStateTest, TestWarmupStateWithZeroSegments) { // Add rowset with zero segments bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(add_result); // State should be immediately ready for completion since there are no segments to warm up WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); - - // Any completion call should handle the edge case gracefully - WarmUpState result = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - // With 0 segments, the counter should already be 0, so this should transition to DONE - EXPECT_EQ(result, WarmUpState::DONE); + EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); } // Test concurrent access to warmup state (basic thread safety verification) @@ -341,26 +341,26 @@ TEST_F(CloudTabletWarmUpStateTest, TestConcurrentWarmupStateAccess) { // Add rowsets from different "threads" (simulated by sequential calls) EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset1->rowset_meta()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN)); + WarmUpTriggerSource::EVENT_DRIVEN)); EXPECT_TRUE(_tablet->add_rowset_warmup_state(*(rowset2->rowset_meta()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET)); + WarmUpTriggerSource::SYNC_ROWSET)); // Interleaved completion operations - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_SYNC_ROWSET, + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET, rowset2->rowset_id(), Status::OK(), 1, 0), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); + EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); // Check states are maintained correctly EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), - WarmUpState::TRIGGERED_BY_EVENT_DRIVEN); + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), - WarmUpState::TRIGGERED_BY_SYNC_ROWSET); + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); } // Test only when the trigger source matches can the state be updated @@ -368,21 +368,21 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupTriggerSource) auto rowset = create_rowset(Version(13, 13), 1); ASSERT_NE(rowset, nullptr); // Add rowset warmup state - bool add_result = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), - WarmUpState::TRIGGERED_BY_JOB); + bool add_result = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); EXPECT_TRUE(add_result); // Attempt to complete with a different trigger source, should not update state WarmUpState result = _tablet->complete_rowset_segment_warmup( - WarmUpState::TRIGGERED_BY_SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpState::TRIGGERED_BY_JOB); - result = _tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_EVENT_DRIVEN, + WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); + EXPECT_EQ(result, WarmUpTriggerSource::JOB); + result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpState::TRIGGERED_BY_JOB); + EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}); // Now complete with the correct trigger source - result = _tablet->complete_rowset_segment_warmup(WarmUpState::TRIGGERED_BY_JOB, - rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpState::DONE); + result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::JOB, rowset->rowset_id(), + Status::OK(), 1, 0); + EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DONE}); } } // namespace doris From 79dc06a13bf4a64821b825431fa46ffee12d81dd Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 9 Oct 2025 16:44:11 +0800 Subject: [PATCH 3/6] tmp2 --- be/src/cloud/cloud_tablet.h | 4 + be/test/cloud/cloud_tablet_test.cpp | 144 +++++++++++++++++----------- 2 files changed, 90 insertions(+), 58 deletions(-) diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 5eb0e7830e868d..4bbda0004bfb54 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -34,6 +34,10 @@ enum class WarmUpProgress : int { NONE, DOING, DONE }; struct WarmUpState { WarmUpTriggerSource trigger_source {WarmUpTriggerSource::NONE}; WarmUpProgress progress {WarmUpProgress::NONE}; + + bool operator==(const WarmUpState& other) const { + return trigger_source == other.trigger_source && progress == other.progress; + } }; struct SyncRowsetStats { diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index ff1fbf3cc833cf..191734b3668a28 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -80,7 +80,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestGetRowsetWarmupStateNonExistent) { auto non_existent_id = _engine.next_rowset_id(); WarmUpState state = _tablet->get_rowset_warmup_state(non_existent_id); - EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::NONE, WarmUpProgress::NONE}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::NONE, WarmUpProgress::NONE}; + EXPECT_EQ(state, expected_state); } // Test add_rowset_warmup_state with TRIGGERED_BY_EVENT_DRIVEN state @@ -94,7 +95,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredByJob) { // Verify the state is correctly set WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); } // Test add_rowset_warmup_state with TRIGGERED_BY_SYNC_ROWSET state @@ -108,7 +110,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredBySyncRowset // Verify the state is correctly set WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); } // Test adding duplicate rowset warmup state should fail @@ -128,7 +131,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddDuplicateRowsetWarmupState) { // State should remain the original one WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); } // Test complete_rowset_segment_warmup for non-existent rowset @@ -137,7 +141,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupNonExistent) { WarmUpState result = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::SYNC_ROWSET, non_existent_id, Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::NONE, WarmUpProgress::NONE}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::NONE, WarmUpProgress::NONE}; + EXPECT_EQ(result, expected_state); } // Test complete_rowset_segment_warmup with partial completion @@ -153,17 +158,19 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupPartial) { // Complete one segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result1, expected_state); // Complete second segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state WarmUpState result2 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result2, expected_state); // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(current_state, - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(current_state, expected_state); } // Test complete_rowset_segment_warmup with full completion @@ -179,16 +186,19 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupFull) { // Complete first segment WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(result1, expected_state); // Complete second segment, should transition to DONE state WarmUpState result2 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DONE}); + expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DONE}; + EXPECT_EQ(result2, expected_state); // Verify final state is DONE WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(final_state, WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DONE}); + expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DONE}; + EXPECT_EQ(final_state, expected_state); } // Test complete_rowset_segment_warmup with inverted index file, partial completion @@ -209,17 +219,19 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Complete one segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result1, expected_state); // Complete inverted index file, should still be in TRIGGERED_BY_EVENT_DRIVEN state WarmUpState result2 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); - EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result2, expected_state); // Verify current state is still TRIGGERED_BY_EVENT_DRIVEN WarmUpState current_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(current_state, - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(current_state, expected_state); } // Test complete_rowset_segment_warmup with inverted index file, full completion @@ -238,16 +250,19 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Complete segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result1, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result1, expected_state); // Complete inverted index file WarmUpState result2 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 0, 1); - EXPECT_EQ(result2, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(result2, expected_state); // Verify final state is DONE WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(final_state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(final_state, expected_state); } // Test complete_rowset_segment_warmup with error status @@ -264,11 +279,13 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithError) { Status error_status = Status::InternalError("Test error"); WarmUpState result = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), error_status, 1, 0); - EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(result, expected_state); // Verify final state is DONE even with error WarmUpState final_state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(final_state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(final_state, expected_state); } // Test multiple rowsets warmup state management @@ -289,32 +306,36 @@ TEST_F(CloudTabletWarmUpStateTest, TestMultipleRowsetsWarmupState) { WarmUpTriggerSource::EVENT_DRIVEN)); // Verify all states - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), - WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), expected_state); + expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), expected_state); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), expected_state); // Complete rowset1 (2 segments) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); + WarmUpState result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result, expected_state); + result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(result, expected_state); // Complete rowset3 (1 segment) - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset3->rowset_id(), Status::OK(), 1, 0), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); + result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, + rowset3->rowset_id(), Status::OK(), 1, 0); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(result, expected_state); // Verify states after completion - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), - WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), WarmUpState::DONE); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), expected_state); + expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), expected_state); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), expected_state); } // Test warmup state with zero segments (edge case) @@ -329,7 +350,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestWarmupStateWithZeroSegments) { // State should be immediately ready for completion since there are no segments to warm up WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - EXPECT_EQ(state, WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(state, expected_state); } // Test concurrent access to warmup state (basic thread safety verification) @@ -346,21 +368,24 @@ TEST_F(CloudTabletWarmUpStateTest, TestConcurrentWarmupStateAccess) { WarmUpTriggerSource::SYNC_ROWSET)); // Interleaved completion operations - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET, - rowset2->rowset_id(), Status::OK(), 1, 0), - WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); - EXPECT_EQ(_tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); + WarmUpState result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result, expected_state); + result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET, + rowset2->rowset_id(), Status::OK(), 1, 0); + expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(result, expected_state); + result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, + rowset1->rowset_id(), Status::OK(), 1, 0); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(result, expected_state); // Check states are maintained correctly - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), - WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}); - EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), - WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), expected_state); + expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), expected_state); } // Test only when the trigger source matches can the state be updated @@ -375,14 +400,17 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupTriggerSource) // Attempt to complete with a different trigger source, should not update state WarmUpState result = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpTriggerSource::JOB); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(result, expected_state); result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(result, expected_state); // Now complete with the correct trigger source result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::JOB, rowset->rowset_id(), Status::OK(), 1, 0); - EXPECT_EQ(result, WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DONE}); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DONE}; + EXPECT_EQ(result, expected_state); } } // namespace doris From d414ea70b36639d5d0a17367d031914a0cda9b53 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 9 Oct 2025 16:55:11 +0800 Subject: [PATCH 4/6] format --- be/test/cloud/cloud_tablet_test.cpp | 49 ++++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index 191734b3668a28..7b21a1b368986c 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -95,7 +95,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredByJob) { // Verify the state is correctly set WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(state, expected_state); } @@ -110,7 +111,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddRowsetWarmupStateTriggeredBySyncRowset // Verify the state is correctly set WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; EXPECT_EQ(state, expected_state); } @@ -131,7 +133,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestAddDuplicateRowsetWarmupState) { // State should remain the original one WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(state, expected_state); } @@ -158,7 +161,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupPartial) { // Complete one segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(result1, expected_state); // Complete second segment, should still be in TRIGGERED_BY_EVENT_DRIVEN state @@ -186,7 +190,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupFull) { // Complete first segment WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; EXPECT_EQ(result1, expected_state); // Complete second segment, should transition to DONE state @@ -219,7 +224,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Complete one segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(result1, expected_state); // Complete inverted index file, should still be in TRIGGERED_BY_EVENT_DRIVEN state @@ -250,7 +256,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn // Complete segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(result1, expected_state); // Complete inverted index file @@ -279,7 +286,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithError) { Status error_status = Status::InternalError("Test error"); WarmUpState result = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), error_status, 1, 0); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; EXPECT_EQ(result, expected_state); // Verify final state is DONE even with error @@ -306,7 +314,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestMultipleRowsetsWarmupState) { WarmUpTriggerSource::EVENT_DRIVEN)); // Verify all states - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset1->rowset_id()), expected_state); expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset2->rowset_id()), expected_state); @@ -314,18 +323,18 @@ TEST_F(CloudTabletWarmUpStateTest, TestMultipleRowsetsWarmupState) { EXPECT_EQ(_tablet->get_rowset_warmup_state(rowset3->rowset_id()), expected_state); // Complete rowset1 (2 segments) - WarmUpState result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0); + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(), Status::OK(), 1, 0); expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(result, expected_state); result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0); + rowset1->rowset_id(), Status::OK(), 1, 0); expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; EXPECT_EQ(result, expected_state); // Complete rowset3 (1 segment) result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset3->rowset_id(), Status::OK(), 1, 0); + rowset3->rowset_id(), Status::OK(), 1, 0); expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; EXPECT_EQ(result, expected_state); @@ -350,7 +359,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestWarmupStateWithZeroSegments) { // State should be immediately ready for completion since there are no segments to warm up WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; EXPECT_EQ(state, expected_state); } @@ -368,16 +378,17 @@ TEST_F(CloudTabletWarmUpStateTest, TestConcurrentWarmupStateAccess) { WarmUpTriggerSource::SYNC_ROWSET)); // Interleaved completion operations - WarmUpState result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0); - WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpTriggerSource::EVENT_DRIVEN, rowset1->rowset_id(), Status::OK(), 1, 0); + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(result, expected_state); result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::SYNC_ROWSET, - rowset2->rowset_id(), Status::OK(), 1, 0); + rowset2->rowset_id(), Status::OK(), 1, 0); expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; EXPECT_EQ(result, expected_state); result = _tablet->complete_rowset_segment_warmup(WarmUpTriggerSource::EVENT_DRIVEN, - rowset1->rowset_id(), Status::OK(), 1, 0); + rowset1->rowset_id(), Status::OK(), 1, 0); expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; EXPECT_EQ(result, expected_state); From f3efcff32e7c856098bf73f49e9ba1abc662cc91 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 9 Oct 2025 17:25:31 +0800 Subject: [PATCH 5/6] add ut by claude --- be/src/cloud/cloud_tablet.cpp | 11 +- be/src/cloud/cloud_tablet.h | 4 +- be/test/cloud/cloud_tablet_test.cpp | 235 +++++++++++++++++++++++++++- 3 files changed, 237 insertions(+), 13 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 0bbb2dc1e98a56..df3fa30006680d 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -1662,24 +1662,25 @@ bool CloudTablet::add_rowset_warmup_state(const RowsetMeta& rowset, WarmUpTrigge return add_rowset_warmup_state_unlocked(rowset, source, start_tp); } -void CloudTablet::update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource source, +bool CloudTablet::update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource source, RowsetId rowset_id, int64_t delta) { std::lock_guard wlock(_meta_lock); - update_rowset_warmup_state_inverted_idx_num_unlocked(source, rowset_id, delta); + return update_rowset_warmup_state_inverted_idx_num_unlocked(source, rowset_id, delta); } -void CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source, +bool CloudTablet::update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source, RowsetId rowset_id, int64_t delta) { auto it = _rowset_warm_up_states.find(rowset_id); if (it == _rowset_warm_up_states.end()) { - return; + return false; } if (it->second.state.trigger_source != source) { // Only the same trigger source can update the state - return; + return false; } it->second.num_inverted_idx += delta; + return true; } bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 4bbda0004bfb54..e485a401221599 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -340,9 +340,9 @@ class CloudTablet final : public BaseTablet { bool add_rowset_warmup_state( const RowsetMeta& rowset, WarmUpTriggerSource source, std::chrono::steady_clock::time_point start_tp = std::chrono::steady_clock::now()); - void update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource source, RowsetId rowset_id, + bool update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource source, RowsetId rowset_id, int64_t delta); - void update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source, + bool update_rowset_warmup_state_inverted_idx_num_unlocked(WarmUpTriggerSource source, RowsetId rowset_id, int64_t delta); WarmUpState complete_rowset_segment_warmup(WarmUpTriggerSource trigger_source, RowsetId rowset_id, Status status, diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index 7b21a1b368986c..abd30b55ceed4c 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -216,10 +216,10 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(add_result); - _tablet->update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource::EVENT_DRIVEN, - rowset->rowset_id(), 1); - _tablet->update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource::EVENT_DRIVEN, - rowset->rowset_id(), 1); + EXPECT_TRUE(_tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1)); + EXPECT_TRUE(_tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1)); // Complete one segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( @@ -250,8 +250,8 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupWithInvertedIn WarmUpTriggerSource::EVENT_DRIVEN); EXPECT_TRUE(add_result); - _tablet->update_rowset_warmup_state_inverted_idx_num(WarmUpTriggerSource::EVENT_DRIVEN, - rowset->rowset_id(), 1); + EXPECT_TRUE(_tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1)); // Complete segment file WarmUpState result1 = _tablet->complete_rowset_segment_warmup( @@ -408,6 +408,14 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupTriggerSource) _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); EXPECT_TRUE(add_result); + // Attempt to update inverted index num with a different trigger source, should fail + bool update_result = _tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), 1); + EXPECT_FALSE(update_result); + update_result = _tablet->update_rowset_warmup_state_inverted_idx_num( + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), 1); + EXPECT_FALSE(update_result); + // Attempt to complete with a different trigger source, should not update state WarmUpState result = _tablet->complete_rowset_segment_warmup( WarmUpTriggerSource::SYNC_ROWSET, rowset->rowset_id(), Status::OK(), 1, 0); @@ -424,4 +432,219 @@ TEST_F(CloudTabletWarmUpStateTest, TestCompleteRowsetSegmentWarmupTriggerSource) expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DONE}; EXPECT_EQ(result, expected_state); } + +// Test JOB trigger source can override non-JOB warmup states +TEST_F(CloudTabletWarmUpStateTest, TestJobTriggerOverridesNonJobStates) { + auto rowset = create_rowset(Version(14, 14), 2); + ASSERT_NE(rowset, nullptr); + + // First, add EVENT_DRIVEN warmup state + bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::EVENT_DRIVEN); + EXPECT_TRUE(result1); + + // Verify EVENT_DRIVEN state is set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); + + // Now add JOB warmup state, should override EVENT_DRIVEN + bool result2 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result2); + + // Verify JOB state has overridden EVENT_DRIVEN + state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + +// Test JOB trigger source can override SYNC_ROWSET warmup state +TEST_F(CloudTabletWarmUpStateTest, TestJobTriggerOverridesSyncRowsetState) { + auto rowset = create_rowset(Version(15, 15), 3); + ASSERT_NE(rowset, nullptr); + + // First, add SYNC_ROWSET warmup state + bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::SYNC_ROWSET); + EXPECT_TRUE(result1); + + // Verify SYNC_ROWSET state is set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); + + // Now add JOB warmup state, should override SYNC_ROWSET + bool result2 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result2); + + // Verify JOB state has overridden SYNC_ROWSET + state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + +// Test JOB trigger source cannot override another JOB warmup state +TEST_F(CloudTabletWarmUpStateTest, TestJobTriggerCannotOverrideAnotherJob) { + auto rowset = create_rowset(Version(16, 16), 2); + ASSERT_NE(rowset, nullptr); + + // First, add JOB warmup state + bool result1 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result1); + + // Verify first JOB state is set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); + + // Try to add another JOB warmup state, should fail + bool result2 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_FALSE(result2); + + // Verify state remains the original JOB state + state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + +// Test EVENT_DRIVEN cannot override existing JOB state +TEST_F(CloudTabletWarmUpStateTest, TestEventDrivenCannotOverrideJobState) { + auto rowset = create_rowset(Version(17, 17), 1); + ASSERT_NE(rowset, nullptr); + + // First, add JOB warmup state + bool result1 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result1); + + // Verify JOB state is set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); + + // Try to add EVENT_DRIVEN warmup state, should fail + bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::EVENT_DRIVEN); + EXPECT_FALSE(result2); + + // Verify state remains JOB + state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + +// Test SYNC_ROWSET cannot override existing JOB state +TEST_F(CloudTabletWarmUpStateTest, TestSyncRowsetCannotOverrideJobState) { + auto rowset = create_rowset(Version(18, 18), 2); + ASSERT_NE(rowset, nullptr); + + // First, add JOB warmup state + bool result1 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result1); + + // Verify JOB state is set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); + + // Try to add SYNC_ROWSET warmup state, should fail + bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::SYNC_ROWSET); + EXPECT_FALSE(result2); + + // Verify state remains JOB + state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + +// Test EVENT_DRIVEN cannot override existing SYNC_ROWSET state +TEST_F(CloudTabletWarmUpStateTest, TestEventDrivenCannotOverrideSyncRowsetState) { + auto rowset = create_rowset(Version(19, 19), 1); + ASSERT_NE(rowset, nullptr); + + // First, add SYNC_ROWSET warmup state + bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::SYNC_ROWSET); + EXPECT_TRUE(result1); + + // Verify SYNC_ROWSET state is set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); + + // Try to add EVENT_DRIVEN warmup state, should fail + bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::EVENT_DRIVEN); + EXPECT_FALSE(result2); + + // Verify state remains SYNC_ROWSET + state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::SYNC_ROWSET, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + +// Test SYNC_ROWSET cannot override existing EVENT_DRIVEN state +TEST_F(CloudTabletWarmUpStateTest, TestSyncRowsetCannotOverrideEventDrivenState) { + auto rowset = create_rowset(Version(20, 20), 3); + ASSERT_NE(rowset, nullptr); + + // First, add EVENT_DRIVEN warmup state + bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::EVENT_DRIVEN); + EXPECT_TRUE(result1); + + // Verify EVENT_DRIVEN state is set + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); + + // Try to add SYNC_ROWSET warmup state, should fail + bool result2 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::SYNC_ROWSET); + EXPECT_FALSE(result2); + + // Verify state remains EVENT_DRIVEN + state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + +// Test JOB can override DONE state from non-JOB source +TEST_F(CloudTabletWarmUpStateTest, TestJobCanOverrideDoneStateFromNonJob) { + auto rowset = create_rowset(Version(21, 21), 1); + ASSERT_NE(rowset, nullptr); + + // First, add EVENT_DRIVEN warmup state + bool result1 = _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), + WarmUpTriggerSource::EVENT_DRIVEN); + EXPECT_TRUE(result1); + + // Complete the warmup to DONE state + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpTriggerSource::EVENT_DRIVEN, rowset->rowset_id(), Status::OK(), 1, 0); + WarmUpState expected_state = + WarmUpState {WarmUpTriggerSource::EVENT_DRIVEN, WarmUpProgress::DONE}; + EXPECT_EQ(result, expected_state); + + // Now add JOB warmup state, should override the DONE state + bool result2 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result2); + + // Verify JOB state has overridden the DONE state + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} + } // namespace doris From 6f27bc248e81ad34c30e0d2f6692734ad1cffc00 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 10 Oct 2025 16:36:04 +0800 Subject: [PATCH 6/6] fix --- be/src/cloud/cloud_tablet.cpp | 8 +++++--- be/test/cloud/cloud_tablet_test.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index df3fa30006680d..ba404d9138b88e 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -1695,11 +1695,11 @@ bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, // For job-triggered warmup (one-time and periodic warmup), allow it to proceed // except when there's already another job-triggered warmup in progress if (source == WarmUpTriggerSource::JOB) { - if (existing_state.trigger_source == WarmUpTriggerSource::JOB) { + if (existing_state.trigger_source == WarmUpTriggerSource::JOB && + existing_state.progress == WarmUpProgress::DOING) { // Same job type already in progress, skip to avoid duplicate warmup return false; } - // Override other states (EVENT_DRIVEN, SYNC_ROWSET) but not another JOB } else { // For non-job warmup (EVENT_DRIVEN, SYNC_ROWSET), skip if any warmup exists return false; @@ -1714,7 +1714,9 @@ bool CloudTablet::add_rowset_warmup_state_unlocked(const RowsetMeta& rowset, g_file_cache_warm_up_rowset_triggered_by_event_driven_num << 1; } _rowset_warm_up_states[rowset_id] = { - .state = {.trigger_source = source, .progress = WarmUpProgress::DOING}, + .state = {.trigger_source = source, + .progress = (rowset.num_segments() == 0 ? WarmUpProgress::DONE + : WarmUpProgress::DOING)}, .num_segments = rowset.num_segments(), .start_tp = start_tp}; return true; diff --git a/be/test/cloud/cloud_tablet_test.cpp b/be/test/cloud/cloud_tablet_test.cpp index abd30b55ceed4c..0e9c39144c684f 100644 --- a/be/test/cloud/cloud_tablet_test.cpp +++ b/be/test/cloud/cloud_tablet_test.cpp @@ -647,4 +647,30 @@ TEST_F(CloudTabletWarmUpStateTest, TestJobCanOverrideDoneStateFromNonJob) { EXPECT_EQ(state, expected_state); } +// Test JOB can override DONE state from JOB source +TEST_F(CloudTabletWarmUpStateTest, TestJobCanOverrideDoneStateFromJob) { + auto rowset = create_rowset(Version(21, 21), 1); + ASSERT_NE(rowset, nullptr); + + // First, add EVENT_DRIVEN warmup state + bool result1 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result1); + + // Complete the warmup to DONE state + WarmUpState result = _tablet->complete_rowset_segment_warmup( + WarmUpTriggerSource::JOB, rowset->rowset_id(), Status::OK(), 1, 0); + WarmUpState expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DONE}; + EXPECT_EQ(result, expected_state); + + // Now add JOB warmup state, should override the DONE state + bool result2 = + _tablet->add_rowset_warmup_state(*(rowset->rowset_meta()), WarmUpTriggerSource::JOB); + EXPECT_TRUE(result2); + + // Verify JOB state has overridden the DONE state + WarmUpState state = _tablet->get_rowset_warmup_state(rowset->rowset_id()); + expected_state = WarmUpState {WarmUpTriggerSource::JOB, WarmUpProgress::DOING}; + EXPECT_EQ(state, expected_state); +} } // namespace doris