Skip to content

[Revert](merge-on-write) Don't use delete bitmap to mark delete for rows with delete sign when sequence column doesn't exist #26721

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 2 commits into from
Nov 10, 2023
Merged

[Revert](merge-on-write) Don't use delete bitmap to mark delete for rows with delete sign when sequence column doesn't exist #26721

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8fda2ff
update
bobhan1 Nov 9, 2023
04c7244
reserve cases
bobhan1 Nov 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 0 additions & 30 deletions be/src/olap/rowset/segment_v2/segment_writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -401,13 +401,6 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block*
// mark key with delete sign as deleted.
bool have_delete_sign =
(delete_sign_column_data != nullptr && delete_sign_column_data[block_pos] != 0);
if (have_delete_sign && !_tablet_schema->has_sequence_col() && !have_input_seq_column) {
// we can directly use delete bitmap to mark the rows with delete sign as deleted
// if sequence column doesn't exist to eliminate reading delete sign columns in later reads
_mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id,
DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
segment_pos);
}

RowLocation loc;
// save rowset shared ptr so this rowset wouldn't delete
Expand Down Expand Up @@ -660,29 +653,6 @@ Status SegmentWriter::append_block(const vectorized::Block* block, size_t row_po
_serialize_block_to_row_column(*const_cast<vectorized::Block*>(block));
}

if (_opts.write_type == DataWriteType::TYPE_DIRECT && _opts.enable_unique_key_merge_on_write &&
!_tablet_schema->has_sequence_col() && _tablet_schema->delete_sign_idx() != -1) {
const vectorized::ColumnWithTypeAndName& delete_sign_column =
block->get_by_position(_tablet_schema->delete_sign_idx());
auto& delete_sign_col =
reinterpret_cast<const vectorized::ColumnInt8&>(*(delete_sign_column.column));
if (delete_sign_col.size() >= row_pos + num_rows) {
const vectorized::Int8* delete_sign_column_data = delete_sign_col.get_data().data();
uint32_t segment_start_pos =
_column_writers[_tablet_schema->delete_sign_idx()]->get_next_rowid();
for (size_t block_pos = row_pos, seg_pos = segment_start_pos;
seg_pos < segment_start_pos + num_rows; block_pos++, seg_pos++) {
// we can directly use delete bitmap to mark the rows with delete sign as deleted
// if sequence column doesn't exist to eliminate reading delete sign columns in later reads
if (delete_sign_column_data[block_pos]) {
_mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id,
DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
seg_pos);
}
}
}
}

_olap_data_convertor->set_source_content(block, row_pos, num_rows);

// find all row pos for short key indexes
Expand Down
8 changes: 0 additions & 8 deletions be/src/olap/tablet.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2978,14 +2978,6 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
continue;
}
if (is_partial_update && rowset_writer != nullptr) {
if (delete_bitmap->contains(
{rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_FOR_DELETE_SIGN},
row_id)) {
LOG(INFO)
<< "DEBUG: skip a delete sign column while calc_segment_delete_bitmap "
<< "processing confict for partial update";
continue;
}
// In publish version, record rows to be deleted for concurrent update
// For example, if version 5 and 6 update a row, but version 6 only see
// version 4 when write, and when publish version, version 5's value will
Expand Down
1 change: 0 additions & 1 deletion be/src/olap/tablet_meta.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,6 @@ class DeleteBitmap {
// tablet's delete bitmap we can use arbitary version number in BitmapKey. Here we define some version numbers
// for specific usage during this periods to avoid conflicts
constexpr static inline uint64_t TEMP_VERSION_COMMON = 0;
constexpr static inline uint64_t TEMP_VERSION_FOR_DELETE_SIGN = 1;

/**
*
Expand Down
37 changes: 1 addition & 36 deletions regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
2 2 2 2 2
4 4 4 4 4

-- !1 --
-- !with_delete_sign --
1 \N \N \N \N 1
1 1 1 1 1 0
2 2 2 2 2 0
Expand All @@ -21,47 +21,12 @@
5 5 5 5 5 0
6 \N \N \N \N 1

-- !2 --
2 2 2 2 2 0
4 4 4 4 4 0

-- !sql --
1 1 1 1 1
2 2 2 2 2
3 3 3 3 3
4 4 4 4 4
5 5 5 5 5

-- !after_delete --
2 2 2 2 2
4 4 4 4 4

-- !1 --
1 \N \N \N \N 1
1 1 1 1 1 0
2 2 2 2 2 0
3 \N \N \N \N 1
3 3 3 3 3 0
4 4 4 4 4 0
5 \N \N \N \N 1
5 5 5 5 5 0
6 \N \N \N \N 1

-- !2 --
1 \N \N \N \N 1
2 2 2 2 2 0
3 \N \N \N \N 1
4 4 4 4 4 0
5 \N \N \N \N 1
6 \N \N \N \N 1

-- !1 --
1 1 1

-- !2 --

-- !3 --
1 2 \N

-- !1 --
1 1 1 1
Expand Down
54 changes: 0 additions & 54 deletions regression-test/data/unique_with_mow_p0/test_delete_sign_delete_bitmap.out
View file
Open in desktop

This file was deleted.

117 changes: 59 additions & 58 deletions ...sion-test/suites/unique_with_mow_p0/partial_update/test_partial_update_delete_sign.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,67 +55,68 @@ suite('test_partial_update_delete_sign') {
sql "set skip_storage_engine_merge=true;"
sql "set skip_delete_bitmap=true;"
sql "sync"
// skip_delete_bitmap=true, skip_delete_sign=true
qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"

sql "set skip_delete_sign=true;"
sql "set skip_delete_bitmap=false;"
sql "sync"
// skip_delete_bitmap=false, skip_delete_sign=true
qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
// // skip_delete_bitmap=true, skip_delete_sign=true
// qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"

// sql "set skip_delete_sign=true;"
// sql "set skip_delete_bitmap=false;"
// sql "sync"
// // skip_delete_bitmap=false, skip_delete_sign=true
// qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
qt_with_delete_sign "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName1} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
sql "drop table if exists ${tableName1};"


sql "set skip_delete_sign=false;"
sql "set skip_storage_engine_merge=false;"
sql "set skip_delete_bitmap=false;"
sql "sync"
def tableName2 = "test_partial_update_delete_sign2"
sql "DROP TABLE IF EXISTS ${tableName2};"
sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
`k1` int NOT NULL,
`c1` int,
`c2` int,
`c3` int,
`c4` int
)UNIQUE KEY(k1)
DISTRIBUTED BY HASH(k1) BUCKETS 1
PROPERTIES (
"enable_unique_key_merge_on_write" = "true",
"disable_auto_compaction" = "true",
"replication_num" = "1",
"function_column.sequence_col" = 'c4'
);"""

sql "insert into ${tableName2} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
streamLoad {
table "${tableName2}"

set 'column_separator', ','
set 'format', 'csv'
set 'partial_columns', 'true'
set 'columns', 'k1,__DORIS_DELETE_SIGN__'

file 'delete_sign.csv'
time 10000 // limit inflight 10s
}
sql "sync"
qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;"

sql "set skip_delete_sign=true;"
sql "set skip_storage_engine_merge=true;"
sql "set skip_delete_bitmap=true;"
sql "sync"
// skip_delete_bitmap=true, skip_delete_sign=true
qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"

sql "set skip_delete_sign=true;"
sql "set skip_delete_bitmap=false;"
sql "sync"
// skip_delete_bitmap=false, skip_delete_sign=true
qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
sql "drop table if exists ${tableName2};"
// sql "set skip_delete_sign=false;"
// sql "set skip_storage_engine_merge=false;"
// sql "set skip_delete_bitmap=false;"
// sql "sync"
// def tableName2 = "test_partial_update_delete_sign2"
// sql "DROP TABLE IF EXISTS ${tableName2};"
// sql """ CREATE TABLE IF NOT EXISTS ${tableName2} (
// `k1` int NOT NULL,
// `c1` int,
// `c2` int,
// `c3` int,
// `c4` int
// )UNIQUE KEY(k1)
// DISTRIBUTED BY HASH(k1) BUCKETS 1
// PROPERTIES (
// "enable_unique_key_merge_on_write" = "true",
// "disable_auto_compaction" = "true",
// "replication_num" = "1",
// "function_column.sequence_col" = 'c4'
// );"""

// sql "insert into ${tableName2} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"
// qt_sql "select * from ${tableName2} order by k1,c1,c2,c3,c4;"
// streamLoad {
// table "${tableName2}"

// set 'column_separator', ','
// set 'format', 'csv'
// set 'partial_columns', 'true'
// set 'columns', 'k1,__DORIS_DELETE_SIGN__'

// file 'delete_sign.csv'
// time 10000 // limit inflight 10s
// }
// sql "sync"
// qt_after_delete "select * from ${tableName2} order by k1,c1,c2,c3,c4;"

// sql "set skip_delete_sign=true;"
// sql "set skip_storage_engine_merge=true;"
// sql "set skip_delete_bitmap=true;"
// sql "sync"
// // skip_delete_bitmap=true, skip_delete_sign=true
// qt_1 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"

// sql "set skip_delete_sign=true;"
// sql "set skip_delete_bitmap=false;"
// sql "sync"
// // skip_delete_bitmap=false, skip_delete_sign=true
// qt_2 "select k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__ from ${tableName2} order by k1,c1,c2,c3,c4,__DORIS_DELETE_SIGN__;"
// sql "drop table if exists ${tableName2};"


// partial update a row that has been deleted by delete sign(table without sequence column)
Expand Down
Loading