Skip to content

[fix](merge-on-write) delete all rows with same key in all pre segments #14995

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 6 commits into from
Dec 19, 2022
Merged

[fix](merge-on-write) delete all rows with same key in all pre segments #14995

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
99ff0bb
debug
nextdreamblue Dec 9, 2022
b2bb1dc
[fix](merge-on-write) delete all rows with same key in all pre segments
nextdreamblue Dec 11, 2022
cfda455
fix code
nextdreamblue Dec 11, 2022
8f2007e
fix code 2
nextdreamblue Dec 12, 2022
175283b
fix bug code
nextdreamblue Dec 12, 2022
5edb629
fix code 2
nextdreamblue Dec 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 27 additions & 24 deletions be/src/olap/tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1999,10 +1999,10 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id,
RowLocation loc;
// first check if exist in pre segment
if (check_pre_segments) {
auto st = _check_pk_in_pre_segments(pre_segments, *key, dummy_version,
delete_bitmap, &loc);
auto st = _check_pk_in_pre_segments(rowset_id, pre_segments, *key,
dummy_version, delete_bitmap, &loc);
if (st.ok()) {
delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first},
delete_bitmap->add({rowset_id, loc.segment_id, dummy_version.first},
loc.row_id);
++row_id;
continue;
Expand All @@ -2012,23 +2012,27 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id,
continue;
}
}
auto st = lookup_row_key(*key, specified_rowset_ids, &loc, dummy_version.first - 1);
CHECK(st.ok() || st.is<NOT_FOUND>() || st.is<ALREADY_EXIST>());
if (st.is<NOT_FOUND>()) {
++row_id;
continue;
}

// sequence id smaller than the previous one, so delete current row
if (st.is<ALREADY_EXIST>()) {
loc.rowset_id = rowset_id;
loc.segment_id = seg->id();
loc.row_id = row_id;
}
if (!specified_rowset_ids->empty()) {
auto st = lookup_row_key(*key, specified_rowset_ids, &loc,
dummy_version.first - 1);
CHECK(st.ok() || st.is<NOT_FOUND>() || st.is<ALREADY_EXIST>());
if (st.is<NOT_FOUND>()) {
++row_id;
continue;
}

// sequence id smaller than the previous one, so delete current row
if (st.is<ALREADY_EXIST>()) {
loc.rowset_id = rowset_id;
loc.segment_id = seg->id();
loc.row_id = row_id;
}

delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first},
loc.row_id);
}
++row_id;
delete_bitmap->add({loc.rowset_id, loc.segment_id, dummy_version.first},
loc.row_id);
}
remaining -= num_read;
}
Expand All @@ -2044,15 +2048,15 @@ Status Tablet::calc_delete_bitmap(RowsetId rowset_id,
}

Status Tablet::_check_pk_in_pre_segments(
const std::vector<segment_v2::SegmentSharedPtr>& pre_segments, const Slice& key,
const Version& version, DeleteBitmapPtr delete_bitmap, RowLocation* loc) {
RowsetId rowset_id, const std::vector<segment_v2::SegmentSharedPtr>& pre_segments,
const Slice& key, const Version& version, DeleteBitmapPtr delete_bitmap, RowLocation* loc) {
for (auto it = pre_segments.rbegin(); it != pre_segments.rend(); ++it) {
auto st = (*it)->lookup_row_key(key, loc);
CHECK(st.ok() || st.is<NOT_FOUND>() || st.is<ALREADY_EXIST>());
if (st.is<NOT_FOUND>()) {
continue;
} else if (st.ok() && _schema->has_sequence_col() &&
delete_bitmap->contains({loc->rowset_id, loc->segment_id, version.first},
delete_bitmap->contains({rowset_id, loc->segment_id, version.first},
loc->row_id)) {
// if has sequence col, we continue to compare the sequence_id of
// all segments, util we find an existing key.
Expand Down Expand Up @@ -2126,10 +2130,9 @@ Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, DeleteBitmapP
for (const auto& to_del : rowset_ids_to_del) {
delete_bitmap->remove({to_del, 0, 0}, {to_del, UINT32_MAX, INT64_MAX});
}
if (!rowset_ids_to_add.empty()) {
RETURN_IF_ERROR(calc_delete_bitmap(rowset->rowset_id(), segments, &rowset_ids_to_add,
delete_bitmap, cur_version - 1, true));
}

RETURN_IF_ERROR(calc_delete_bitmap(rowset->rowset_id(), segments, &rowset_ids_to_add,
delete_bitmap, cur_version - 1, true));

// update version without write lock, compaction and publish_txn
// will update delete bitmap, handle compaction with _rowset_update_lock
Expand Down
3 changes: 2 additions & 1 deletion be/src/olap/tablet.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,8 @@ class Tablet : public BaseTablet {
bool _reconstruct_version_tracker_if_necessary();
void _init_context_common_fields(RowsetWriterContext& context);

Status _check_pk_in_pre_segments(const std::vector<segment_v2::SegmentSharedPtr>& pre_segments,
Status _check_pk_in_pre_segments(RowsetId rowset_id,
const std::vector<segment_v2::SegmentSharedPtr>& pre_segments,
const Slice& key, const Version& version,
DeleteBitmapPtr delete_bitmap, RowLocation* loc);
void _rowset_ids_difference(const RowsetIdUnorderedSet& cur, const RowsetIdUnorderedSet& pre,
Expand Down