Skip to content

branch-3.0: [improve](move-memtable) improve error log and message for "not enough streams" #47470 #47721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Feb 24, 2025
Merged

branch-3.0: [improve](move-memtable) improve error log and message for "not enough streams" #47470 #47721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions be/src/vec/sink/writer/vtablet_writer_v2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,7 @@ void VTabletWriterV2::_generate_rows_for_tablet(std::vector<RowPartTabletIds>& r

Status VTabletWriterV2::_select_streams(int64_t tablet_id, int64_t partition_id, int64_t index_id,
std::vector<std::shared_ptr<LoadStreamStub>>& streams) {
std::vector<int64_t> failed_node_ids;
const auto* location = _location->find_tablet(tablet_id);
DBUG_EXECUTE_IF("VTabletWriterV2._select_streams.location_null", { location = nullptr; });
if (location == nullptr) {
Expand All @@ -397,6 +398,9 @@ Status VTabletWriterV2::_select_streams(int64_t tablet_id, int64_t partition_id,
<< ", stream_ok=" << (stream == nullptr ? "no" : "yes");
});
if (stream == nullptr) {
LOG(WARNING) << "skip writing tablet " << tablet_id << " to backend " << node_id
<< ": stream is not open";
failed_node_ids.push_back(node_id);
continue;
}
streams.emplace_back(std::move(stream));
Expand All @@ -407,8 +411,21 @@ Status VTabletWriterV2::_select_streams(int64_t tablet_id, int64_t partition_id,
<< ", num_nodes=" << location->node_ids.size();
});
if (streams.size() <= location->node_ids.size() / 2) {
return Status::InternalError("not enough streams {}/{}", streams.size(),
location->node_ids.size());
std::ostringstream success_msg;
std::ostringstream failed_msg;
for (auto& s : streams) {
success_msg << ", " << s->dst_id();
}
for (auto id : failed_node_ids) {
failed_msg << ", " << id;
}
LOG(INFO) << "failed to write enough replicas " << streams.size() << "/"
<< location->node_ids.size() << " for tablet " << tablet_id
<< " due to connection errors; success nodes" << success_msg.str()
<< "; failed nodes" << failed_msg.str() << ".";
return Status::InternalError(
"failed to write enough replicas {}/{} for tablet {} due to connection errors",
streams.size(), location->node_ids.size(), tablet_id);
}
Status st;
for (auto& stream : streams) {
Expand Down
2 changes: 1 addition & 1 deletion regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ suite("test_multi_replica_fault_injection", "nonConcurrent") {
// test one backend open failure
load_with_injection("VTabletWriterV2._open_streams.skip_one_backend", "success", true)
// test two backend open failure
load_with_injection("VTabletWriterV2._open_streams.skip_two_backends", "not enough streams 1/3", false, "succ replica num 1 < load required replica num 2")
load_with_injection("VTabletWriterV2._open_streams.skip_two_backends", "failed to write enough replicas 1/3 for tablet", false, "succ replica num 1 < load required replica num 2")
sql """ set enable_memtable_on_sink_node=false """
}
}
Loading