Skip to content

[fix](merge-on-write) fix cu compaction correctness check #17347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 1 commit into from
Mar 6, 2023
Merged

[fix](merge-on-write) fix cu compaction correctness check #17347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions be/src/olap/compaction.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -434,11 +434,13 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) {
RETURN_IF_ERROR(_tablet->check_rowid_conversion(_output_rowset, location_map));

if (compaction_type() == READER_CUMULATIVE_COMPACTION) {
std::string err_msg =
"The merged rows is not equal to missed rows in rowid conversion";
DCHECK(stats != nullptr || stats->merged_rows == missed_rows) << err_msg;
std::string err_msg = fmt::format(
"cumulative compaction: the merged rows({}) is not equal to missed "
"rows({}) in rowid conversion, tablet_id: {}, table_id:{}",
stats->merged_rows, missed_rows, _tablet->tablet_id(), _tablet->table_id());
DCHECK(stats == nullptr || stats->merged_rows == missed_rows) << err_msg;
if (stats != nullptr && stats->merged_rows != missed_rows) {
return Status::InternalError(err_msg);
LOG(WARNING) << err_msg;
}
}

Expand Down
6 changes: 3 additions & 3 deletions be/src/olap/tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2567,7 +2567,7 @@ uint64_t Tablet::calc_compaction_output_rowset_delete_bitmap(
uint64_t start_version, uint64_t end_version,
std::map<RowsetSharedPtr, std::list<std::pair<RowLocation, RowLocation>>>* location_map,
DeleteBitmap* output_rowset_delete_bitmap) {
uint64_t missed_rows = 0;
std::set<RowLocation> missed_rows;
RowLocation src;
RowLocation dst;
for (auto& rowset : input_rowsets) {
Expand All @@ -2589,7 +2589,7 @@ uint64_t Tablet::calc_compaction_output_rowset_delete_bitmap(
<< " src loaction: |" << src.rowset_id << "|"
<< src.segment_id << "|" << src.row_id
<< " version: " << cur_version;
missed_rows++;
missed_rows.insert(src);
continue;
}
VLOG_DEBUG << "calc_compaction_output_rowset_delete_bitmap dst location: |"
Expand All @@ -2604,7 +2604,7 @@ uint64_t Tablet::calc_compaction_output_rowset_delete_bitmap(
}
}
}
return missed_rows;
return missed_rows.size();
}

void Tablet::merge_delete_bitmap(const DeleteBitmap& delete_bitmap) {
Expand Down
14 changes: 14 additions & 0 deletions be/src/olap/utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,20 @@ struct RowLocation {
RowsetId rowset_id;
uint32_t segment_id;
uint32_t row_id;

bool operator==(const RowLocation& rhs) const {
return rowset_id == rhs.rowset_id && segment_id == rhs.segment_id && row_id == rhs.row_id;
}

bool operator<(const RowLocation& rhs) const {
if (rowset_id != rhs.rowset_id) {
return rowset_id < rhs.rowset_id;
} else if (segment_id != rhs.segment_id) {
return segment_id < rhs.segment_id;
} else {
return row_id < rhs.row_id;
}
}
};

struct GlobalRowLoacation {
Expand Down