diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index daf28c1620895b..9ebdc9663473a1 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -832,7 +832,6 @@ Status BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, << ", tablet: " << tablet_id() << " rowset: " << rowset_id << " seg_id: " << seg->id() << " dummy_version: " << end_version + 1 << " rows: " << seg->num_rows() << " conflict rows: " << conflict_rows - << " filtered rows: " << rids_be_overwritten.size() << " new generated rows: " << new_generated_rows << " bitmap num: " << delete_bitmap->get_delete_bitmap_count() << " bitmap cardinality: " << delete_bitmap->cardinality() @@ -1005,6 +1004,13 @@ Status BaseTablet::generate_new_block_for_partial_update( auto old_block = rowset_schema->create_block_by_cids(missing_cids); auto update_block = rowset_schema->create_block_by_cids(update_cids); + bool have_input_seq_column = false; + if (rowset_schema->has_sequence_col()) { + have_input_seq_column = + (std::find(update_cids.cbegin(), update_cids.cend(), + rowset_schema->sequence_col_idx()) != update_cids.cend()); + } + // rowid in the final block(start from 0, increase continuously) -> rowid to read in update_block std::map read_index_update; @@ -1058,10 +1064,25 @@ Status BaseTablet::generate_new_block_for_partial_update( // before, even the `strict_mode` is true (which requires partial update // load job can't insert new keys), this "new" key MUST be written into // the new generated segment file. - if (new_block_delete_signs != nullptr && new_block_delete_signs[idx]) { + bool use_default = false; + bool new_row_delete_sign = + (new_block_delete_signs != nullptr && new_block_delete_signs[idx]); + bool old_row_delete_sign = (old_block_delete_signs != nullptr && + old_block_delete_signs[read_index_old[idx]] != 0); + if (old_row_delete_sign) { + if (!rowset_schema->has_sequence_col()) { + use_default = true; + } else if (have_input_seq_column || !rs_column.is_seqeunce_col()) { + // to keep the sequence column value not decreasing, we should read values of seq column + // from old rows even if the old row is deleted when the input don't specify the sequence column, otherwise + // it may cause the merge-on-read based compaction to produce incorrect results + use_default = true; + } + } + + if (new_row_delete_sign) { mutable_column->insert_default(); - } else if (old_block_delete_signs != nullptr && - old_block_delete_signs[read_index_old[idx]] != 0) { + } else if (use_default) { if (rs_column.has_default_value()) { mutable_column->insert_from(*default_value_block.get_by_position(i).column, 0); } else if (rs_column.is_nullable()) { @@ -1090,6 +1111,10 @@ Status BaseTablet::generate_new_block_for_flexible_partial_update( vectorized::Block* output_block) { CHECK(output_block); + int32_t seq_col_unique_id = -1; + if (rowset_schema->has_sequence_col()) { + seq_col_unique_id = rowset_schema->column(rowset_schema->sequence_col_idx()).unique_id(); + } const auto& non_sort_key_cids = partial_update_info->missing_cids; std::vector all_cids(rowset_schema->num_columns()); std::iota(all_cids.begin(), all_cids.end(), 0); @@ -1139,57 +1164,68 @@ Status BaseTablet::generate_new_block_for_flexible_partial_update( update_block.get_by_position(skip_bitmap_col_idx).column->get_ptr().get()) ->get_data()); - VLOG_DEBUG << fmt::format( - "BaseTablet::generate_new_block_for_flexible_partial_update: " - "rids_be_overwritten.size()={}", - rids_be_overwritten.size()); if (rowset_schema->has_sequence_col() && !rids_be_overwritten.empty()) { - int32_t seq_col_unique_id = - rowset_schema->column(rowset_schema->sequence_col_idx()).unique_id(); // If the row specifies the sequence column, we should delete the current row becase the // flexible partial update on the current row has been `overwritten` by the previous one with larger sequence // column value. for (auto it = rids_be_overwritten.begin(); it != rids_be_overwritten.end();) { auto rid = *it; if (!skip_bitmaps->at(rid).contains(seq_col_unique_id)) { - VLOG_DEBUG << fmt::format( - "BaseTablet::generate_new_block_for_flexible_partial_update: rid={} " - "filtered", - rid); ++it; } else { it = rids_be_overwritten.erase(it); - VLOG_DEBUG << fmt::format( - "BaseTablet::generate_new_block_for_flexible_partial_update: rid={} " - "keeped", - rid); } } } - auto fill_one_cell = [&read_index_old](const TabletColumn& tablet_column, std::size_t idx, - vectorized::MutableColumnPtr& new_col, - const vectorized::IColumn& default_value_col, - const vectorized::IColumn& old_value_col, - const vectorized::IColumn& cur_col, bool skipped, - const signed char* delete_sign_column_data) { + auto fill_one_cell = [&read_index_old, &read_index_update, &rowset_schema, partial_update_info]( + const TabletColumn& tablet_column, std::size_t idx, + vectorized::MutableColumnPtr& new_col, + const vectorized::IColumn& default_value_col, + const vectorized::IColumn& old_value_col, + const vectorized::IColumn& cur_col, bool skipped, + bool row_has_sequence_col, + const signed char* delete_sign_column_data) { if (skipped) { - if (delete_sign_column_data != nullptr && - delete_sign_column_data[read_index_old[cast_set(idx)]] != 0) { + bool use_default = false; + bool old_row_delete_sign = + (delete_sign_column_data != nullptr && + delete_sign_column_data[read_index_old[cast_set(idx)]] != 0); + if (old_row_delete_sign) { + if (!rowset_schema->has_sequence_col()) { + use_default = true; + } else if (row_has_sequence_col || + (!tablet_column.is_seqeunce_col() && + (tablet_column.unique_id() != + partial_update_info->sequence_map_col_uid()))) { + // to keep the sequence column value not decreasing, we should read values of seq column(and seq map column) + // from old rows even if the old row is deleted when the input don't specify the sequence column, otherwise + // it may cause the merge-on-read based compaction to produce incorrect results + use_default = true; + } + } + if (use_default) { if (tablet_column.has_default_value()) { new_col->insert_from(default_value_col, 0); } else if (tablet_column.is_nullable()) { assert_cast( new_col.get()) - ->insert_default(); + ->insert_many_defaults(1); + } else if (tablet_column.is_auto_increment()) { + // For auto-increment column, its default value(generated value) is filled in current block in flush phase + // when the load doesn't specify the auto-increment column + // - if the previous conflicting row is deleted, we should use the value in current block as its final value + // - if the previous conflicting row is an insert, we should use the value in old block as its final value to + // keep consistency between replicas + new_col->insert_from(cur_col, read_index_update[cast_set(idx)]); } else { new_col->insert(tablet_column.get_vec_type()->get_default()); } } else { - new_col->insert_from(old_value_col, idx); + new_col->insert_from(old_value_col, read_index_old[cast_set(idx)]); } } else { - new_col->insert_from(cur_col, idx); + new_col->insert_from(cur_col, read_index_update[cast_set(idx)]); } }; @@ -1200,7 +1236,7 @@ Status BaseTablet::generate_new_block_for_flexible_partial_update( auto col_uid = rs_column.unique_id(); for (auto idx = 0; idx < update_rows; ++idx) { if (cid < rowset_schema->num_key_columns()) { - new_col->insert_from(cur_col, idx); + new_col->insert_from(cur_col, read_index_update[idx]); } else { const vectorized::IColumn& default_value_col = *default_value_block.get_by_position(cid - rowset_schema->num_key_columns()) @@ -1208,10 +1244,13 @@ Status BaseTablet::generate_new_block_for_flexible_partial_update( const vectorized::IColumn& old_value_col = *old_block.get_by_position(cid - rowset_schema->num_key_columns()).column; if (rids_be_overwritten.contains(idx)) { - new_col->insert_from(old_value_col, idx); + new_col->insert_from(old_value_col, read_index_old[idx]); } else { fill_one_cell(rs_column, idx, new_col, default_value_col, old_value_col, cur_col, skip_bitmaps->at(idx).contains(col_uid), + rowset_schema->has_sequence_col() + ? !skip_bitmaps->at(idx).contains(seq_col_unique_id) + : false, old_block_delete_signs); } } diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 8255b59787707c..532fb0f4da689e 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -217,8 +217,10 @@ Status MemTable::insert(const vectorized::Block* input_block, } if (_partial_update_mode == UniqueKeyUpdateModePB::UPDATE_FLEXIBLE_COLUMNS && _tablet_schema->has_skip_bitmap_col()) { - // init of _skip_bitmap_col_idx must be before _init_agg_functions() + // init of _skip_bitmap_col_idx and _delete_sign_col_idx must be before _init_agg_functions() _skip_bitmap_col_idx = _tablet_schema->skip_bitmap_col_idx(); + _delete_sign_col_idx = _tablet_schema->delete_sign_idx(); + _delete_sign_col_unique_id = _tablet_schema->column(_delete_sign_col_idx).unique_id(); if (_seq_col_idx_in_block != -1) { _seq_col_unique_id = _tablet_schema->column(_seq_col_idx_in_block).unique_id(); } @@ -457,6 +459,28 @@ void MemTable::_finalize_one_row(RowInBlock* row, } } +void MemTable::_init_row_for_agg(RowInBlock* row, vectorized::MutableBlock& mutable_block) { + row->init_agg_places(_arena.aligned_alloc(_total_size_of_aggregate_states, 16), + _offsets_of_aggregate_states.data()); + for (auto cid = _tablet_schema->num_key_columns(); cid < _num_columns; cid++) { + auto* col_ptr = mutable_block.mutable_columns()[cid].get(); + auto* data = row->agg_places(cid); + _agg_functions[cid]->create(data); + _agg_functions[cid]->add(data, const_cast(&col_ptr), + row->_row_pos, _arena); + } +} +void MemTable::_clear_row_agg(RowInBlock* row) { + if (row->has_init_agg()) { + for (size_t i = _tablet_schema->num_key_columns(); i < _num_columns; ++i) { + auto function = _agg_functions[i]; + auto* agg_place = row->agg_places(i); + function->destroy(agg_place); + } + row->remove_init_agg(); + } +} + template void MemTable::_aggregate() { SCOPED_RAW_TIMER(&_stat.agg_ns); @@ -468,28 +492,16 @@ void MemTable::_aggregate() { auto& block_data = in_block.get_columns_with_type_and_name(); DorisVector> temp_row_in_blocks; temp_row_in_blocks.reserve(_last_sorted_pos); - RowInBlock* prev_row = nullptr; - int row_pos = -1; //only init agg if needed - auto init_for_agg = [&](RowInBlock* row) { - row->init_agg_places(_arena.aligned_alloc(_total_size_of_aggregate_states, 16), - _offsets_of_aggregate_states.data()); - for (auto cid = _tablet_schema->num_key_columns(); cid < _num_columns; cid++) { - auto* col_ptr = mutable_block.mutable_columns()[cid].get(); - auto* data = prev_row->agg_places(cid); - _agg_functions[cid]->create(data); - _agg_functions[cid]->add(data, const_cast(&col_ptr), - prev_row->_row_pos, _arena); - } - }; - - if (!has_skip_bitmap_col || _seq_col_idx_in_block == -1) { + if constexpr (!has_skip_bitmap_col) { + RowInBlock* prev_row = nullptr; + int row_pos = -1; for (const auto& cur_row_ptr : *_row_in_blocks) { RowInBlock* cur_row = cur_row_ptr.get(); if (!temp_row_in_blocks.empty() && (*_vec_row_comparator)(prev_row, cur_row) == 0) { if (!prev_row->has_init_agg()) { - init_for_agg(prev_row); + _init_row_for_agg(prev_row, mutable_block); } _stat.merged_rows++; _aggregate_two_row_in_block(mutable_block, cur_row, prev_row); @@ -509,70 +521,14 @@ void MemTable::_aggregate() { _finalize_one_row(temp_row_in_blocks.back().get(), block_data, row_pos); } } else { - // For flexible partial update and the table has sequence column, considering the following situation: - // there are multiple rows with the same keys in memtable, some of them specify the sequence column, - // some of them don't. We can't do the de-duplication in memtable becasue we can only know the value - // of the sequence column of the row which don't specify seqeuence column in SegmentWriter after we - // probe the historical data. So at here we can only merge rows that have sequence column together and - // merge rows without sequence column together, and finally, perform deduplication on them in SegmentWriter. - - // !!ATTENTION!!: there may be rows with the same keys after MemTable::_aggregate() in this situation. - RowInBlock* row_with_seq_col = nullptr; - int row_pos_with_seq = -1; - RowInBlock* row_without_seq_col = nullptr; - int row_pos_without_seq = -1; - - auto finalize_rows = [&]() { - if (row_with_seq_col != nullptr) { - _finalize_one_row(row_with_seq_col, block_data, row_pos_with_seq); - row_with_seq_col = nullptr; - } - if (row_without_seq_col != nullptr) { - _finalize_one_row(row_without_seq_col, block_data, row_pos_without_seq); - row_without_seq_col = nullptr; - } - }; - auto add_row = [&](const std::shared_ptr& row_ptr, bool with_seq_col) { - RowInBlock* row = row_ptr.get(); - temp_row_in_blocks.push_back(row_ptr); - row_pos++; - if (with_seq_col) { - row_with_seq_col = row; - row_pos_with_seq = row_pos; - } else { - row_without_seq_col = row; - row_pos_without_seq = row_pos; - } - }; - auto& skip_bitmaps = assert_cast( - mutable_block.mutable_columns()[_skip_bitmap_col_idx].get()) - ->get_data(); - for (const auto& cur_row_ptr : *_row_in_blocks) { - RowInBlock* cur_row = cur_row_ptr.get(); - const BitmapValue& skip_bitmap = skip_bitmaps[cur_row->_row_pos]; - bool with_seq_col = !skip_bitmap.contains(_seq_col_unique_id); - // compare keys, the keys of row_with_seq_col and row_with_seq_col is the same, - // choose any of them if it's valid - prev_row = (row_with_seq_col == nullptr) ? row_without_seq_col : row_with_seq_col; - if (prev_row != nullptr && (*_vec_row_comparator)(prev_row, cur_row) == 0) { - prev_row = (with_seq_col ? row_with_seq_col : row_without_seq_col); - if (prev_row == nullptr) { - add_row(cur_row_ptr, with_seq_col); - continue; - } - if (!prev_row->has_init_agg()) { - init_for_agg(prev_row); - } - _stat.merged_rows++; - _aggregate_two_row_in_block(mutable_block, cur_row, prev_row); - } else { - // no more rows to merge for prev rows, finalize them - finalize_rows(); - add_row(cur_row_ptr, with_seq_col); - } + DCHECK(_delete_sign_col_idx != -1); + if (_seq_col_idx_in_block == -1) { + _aggregate_for_flexible_partial_update_without_seq_col( + block_data, mutable_block, temp_row_in_blocks); + } else { + _aggregate_for_flexible_partial_update_with_seq_col(block_data, mutable_block, + temp_row_in_blocks); } - // finalize the last lows - finalize_rows(); } if constexpr (!is_final) { // if is not final, we collect the agg results to input_block and then continue to insert @@ -587,6 +543,99 @@ void MemTable::_aggregate() { } } +template +void MemTable::_aggregate_for_flexible_partial_update_without_seq_col( + const vectorized::ColumnsWithTypeAndName& block_data, + vectorized::MutableBlock& mutable_block, + DorisVector>& temp_row_in_blocks) { + std::shared_ptr prev_row {nullptr}; + int row_pos = -1; + auto& skip_bitmaps = assert_cast( + mutable_block.mutable_columns()[_skip_bitmap_col_idx].get()) + ->get_data(); + auto& delete_signs = assert_cast( + mutable_block.mutable_columns()[_delete_sign_col_idx].get()) + ->get_data(); + std::shared_ptr row_with_delete_sign {nullptr}; + std::shared_ptr row_without_delete_sign {nullptr}; + + auto finalize_rows = [&]() { + if (row_with_delete_sign != nullptr) { + temp_row_in_blocks.push_back(row_with_delete_sign); + _finalize_one_row(row_with_delete_sign.get(), block_data, ++row_pos); + row_with_delete_sign = nullptr; + } + if (row_without_delete_sign != nullptr) { + temp_row_in_blocks.push_back(row_without_delete_sign); + _finalize_one_row(row_without_delete_sign.get(), block_data, ++row_pos); + row_without_delete_sign = nullptr; + } + // _arena.clear(); + }; + + auto add_row = [&](std::shared_ptr row, bool with_delete_sign) { + if (with_delete_sign) { + row_with_delete_sign = std::move(row); + } else { + row_without_delete_sign = std::move(row); + } + }; + for (const auto& cur_row_ptr : *_row_in_blocks) { + RowInBlock* cur_row = cur_row_ptr.get(); + const BitmapValue& skip_bitmap = skip_bitmaps[cur_row->_row_pos]; + bool cur_row_has_delete_sign = (!skip_bitmap.contains(_delete_sign_col_unique_id) && + delete_signs[cur_row->_row_pos] != 0); + prev_row = + (row_with_delete_sign == nullptr) ? row_without_delete_sign : row_with_delete_sign; + // compare keys, the keys of row_with_delete_sign and row_without_delete_sign is the same, + // choose any of them if it's valid + if (prev_row != nullptr && (*_vec_row_comparator)(prev_row.get(), cur_row) == 0) { + if (cur_row_has_delete_sign) { + if (row_without_delete_sign != nullptr) { + // if there exits row without delete sign, remove it first + _clear_row_agg(row_without_delete_sign.get()); + _stat.merged_rows++; + row_without_delete_sign = nullptr; + } + // and then unconditionally replace the previous row + prev_row = row_with_delete_sign; + } else { + prev_row = row_without_delete_sign; + } + + if (prev_row == nullptr) { + add_row(cur_row_ptr, cur_row_has_delete_sign); + } else { + if (!prev_row->has_init_agg()) { + _init_row_for_agg(prev_row.get(), mutable_block); + } + _stat.merged_rows++; + _aggregate_two_row_in_block(mutable_block, cur_row, prev_row.get()); + } + } else { + finalize_rows(); + add_row(cur_row_ptr, cur_row_has_delete_sign); + } + } + // finalize the last lows + finalize_rows(); +} + +template +void MemTable::_aggregate_for_flexible_partial_update_with_seq_col( + const vectorized::ColumnsWithTypeAndName& block_data, + vectorized::MutableBlock& mutable_block, + DorisVector>& temp_row_in_blocks) { + // For flexible partial update, when table has sequence column, we don't do any aggregation + // in memtable. These duplicate rows will be aggregated in VerticalSegmentWriter + int row_pos = -1; + for (const auto& row_ptr : *_row_in_blocks) { + RowInBlock* row = row_ptr.get(); + temp_row_in_blocks.push_back(row_ptr); + _finalize_one_row(row, block_data, ++row_pos); + } +} + void MemTable::shrink_memtable_by_agg() { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER( _resource_ctx->memory_context()->mem_tracker()->write_tracker()); diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index 354438e347b603..f582c26c029201 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -254,8 +254,24 @@ class MemTable { template void _finalize_one_row(RowInBlock* row, const vectorized::ColumnsWithTypeAndName& block_data, int row_pos); + void _init_row_for_agg(RowInBlock* row, vectorized::MutableBlock& mutable_block); + void _clear_row_agg(RowInBlock* row); + template void _aggregate(); + + template + void _aggregate_for_flexible_partial_update_without_seq_col( + const vectorized::ColumnsWithTypeAndName& block_data, + vectorized::MutableBlock& mutable_block, + DorisVector>& temp_row_in_blocks); + + template + void _aggregate_for_flexible_partial_update_with_seq_col( + const vectorized::ColumnsWithTypeAndName& block_data, + vectorized::MutableBlock& mutable_block, + DorisVector>& temp_row_in_blocks); + Status _put_into_output(vectorized::Block& in_block); bool _is_first_insertion; @@ -266,8 +282,10 @@ class MemTable { std::unique_ptr>> _row_in_blocks; size_t _num_columns; - int32_t _seq_col_idx_in_block = -1; + int32_t _seq_col_idx_in_block {-1}; int32_t _skip_bitmap_col_idx {-1}; + int32_t _delete_sign_col_idx {-1}; + int32_t _delete_sign_col_unique_id {-1}; int32_t _seq_col_unique_id {-1}; bool _is_partial_update_and_auto_inc = false; diff --git a/be/src/olap/partial_update_info.cpp b/be/src/olap/partial_update_info.cpp index 093b70e4ebf25e..6aece0aa5d555f 100644 --- a/be/src/olap/partial_update_info.cpp +++ b/be/src/olap/partial_update_info.cpp @@ -25,11 +25,14 @@ #include "olap/olap_common.h" #include "olap/rowset/rowset.h" #include "olap/rowset/rowset_writer_context.h" +#include "olap/rowset/segment_v2/vertical_segment_writer.h" +#include "olap/tablet_meta.h" #include "olap/tablet_schema.h" #include "olap/utils.h" #include "util/bitmap_value.h" #include "vec/common/assert_cast.h" #include "vec/core/block.h" +#include "vec/olap/olap_data_convertor.h" namespace doris { @@ -220,7 +223,7 @@ Status PartialUpdateInfo::handle_new_key(const TabletSchema& tablet_schema, for (auto cid : missing_cids) { const TabletColumn& col = tablet_schema.column(cid); if (skip_bitmap->contains(col.unique_id()) && !col.has_default_value() && - !col.is_nullable() && col.is_auto_increment()) { + !col.is_nullable() && !col.is_auto_increment()) { error_column = col.name(); can_insert_new_row = false; break; @@ -288,6 +291,10 @@ void PartialUpdateInfo::_generate_default_values_for_missing_cids( CHECK_EQ(missing_cids.size(), default_values.size()); } +bool FixedReadPlan::empty() const { + return plan.empty(); +} + void FixedReadPlan::prepare_to_read(const RowLocation& row_location, size_t pos) { plan[row_location.rowset_id][row_location.segment_id].emplace_back(row_location.row_id, pos); } @@ -325,7 +332,7 @@ Status FixedReadPlan::read_columns_by_plan( (*read_index)[pos] = read_idx++; } if (has_row_column) { - auto st = doris::BaseTablet::fetch_value_through_row_column( + auto st = BaseTablet::fetch_value_through_row_column( rowset_iter->second, tablet_schema, segment_id, rids, cids_to_read, block); if (!st.ok()) { LOG(WARNING) << "failed to fetch value through row column"; @@ -357,6 +364,14 @@ Status FixedReadPlan::fill_missing_columns( auto mutable_full_columns = full_block.mutate_columns(); // create old value columns const auto& missing_cids = rowset_ctx->partial_update_info->missing_cids; + bool have_input_seq_column = false; + if (tablet_schema.has_sequence_col()) { + const std::vector& including_cids = rowset_ctx->partial_update_info->update_cids; + have_input_seq_column = + (std::find(including_cids.cbegin(), including_cids.cend(), + tablet_schema.sequence_col_idx()) != including_cids.cend()); + } + auto old_value_block = tablet_schema.create_block_by_cids(missing_cids); CHECK_EQ(missing_cids.size(), old_value_block.columns()); @@ -376,31 +391,40 @@ Status FixedReadPlan::fill_missing_columns( // fill all missing value from mutable_old_columns, need to consider default value and null value for (auto idx = 0; idx < use_default_or_null_flag.size(); idx++) { - // `use_default_or_null_flag[idx] == false` doesn't mean that we should read values from the old row - // for the missing columns. For example, if a table has sequence column, the rows with DELETE_SIGN column - // marked will not be marked in delete bitmap(see https://github.com/apache/doris/pull/24011), so it will - // be found in Tablet::lookup_row_key() and `use_default_or_null_flag[idx]` will be false. But we should not - // read values from old rows for missing values in this occasion. So we should read the DELETE_SIGN column - // to check if a row REALLY exists in the table. auto segment_pos = idx + segment_start_pos; auto pos_in_old_block = read_index[segment_pos]; - if (use_default_or_null_flag[idx] || - (old_delete_signs != nullptr && old_delete_signs[pos_in_old_block] != 0)) { - for (auto i = 0; i < missing_cids.size(); ++i) { - // if the column has default value, fill it with default value - // otherwise, if the column is nullable, fill it with null value - const auto& tablet_column = tablet_schema.column(missing_cids[i]); - auto& missing_col = mutable_full_columns[missing_cids[i]]; + + for (auto i = 0; i < missing_cids.size(); ++i) { + // if the column has default value, fill it with default value + // otherwise, if the column is nullable, fill it with null value + const auto& tablet_column = tablet_schema.column(missing_cids[i]); + auto& missing_col = mutable_full_columns[missing_cids[i]]; + + bool should_use_default = use_default_or_null_flag[idx]; + if (!should_use_default) { + bool old_row_delete_sign = + (old_delete_signs != nullptr && old_delete_signs[pos_in_old_block] != 0); + if (old_row_delete_sign) { + if (!tablet_schema.has_sequence_col()) { + should_use_default = true; + } else if (have_input_seq_column || (!tablet_column.is_seqeunce_col())) { + // to keep the sequence column value not decreasing, we should read values of seq column + // from old rows even if the old row is deleted when the input don't specify the sequence column, otherwise + // it may cause the merge-on-read based compaction to produce incorrect results + should_use_default = true; + } + } + } + + if (should_use_default) { // clang-format off if (tablet_column.has_default_value()) { missing_col->insert_from(*mutable_default_value_columns[i], 0); } else if (tablet_column.is_nullable()) { - auto* nullable_column = - assert_cast(missing_col.get()); - nullable_column->insert_default(); + auto* nullable_column = assert_cast(missing_col.get()); + nullable_column->insert_many_defaults(1); } else if (tablet_schema.auto_increment_column() == tablet_column.name()) { - const auto& column = - *DORIS_TRY(rowset_ctx->tablet_schema->column(tablet_column.name())); + const auto& column = *DORIS_TRY(rowset_ctx->tablet_schema->column(tablet_column.name())); DCHECK(column.type() == FieldType::OLAP_FIELD_TYPE_BIGINT); auto* auto_inc_column = assert_cast(missing_col.get()); @@ -414,12 +438,10 @@ block->get_by_name(BeConsts::PARTIAL_UPDATE_AUTO_INC_COL).column.get())->get_ele missing_col->insert(tablet_column.get_vec_type()->get_default()); } // clang-format on + } else { + missing_col->insert_from(*old_value_block.get_by_position(i).column, + pos_in_old_block); } - continue; - } - for (auto i = 0; i < missing_cids.size(); ++i) { - mutable_full_columns[missing_cids[i]]->insert_from( - *old_value_block.get_by_position(i).column, pos_in_old_block); } } full_block.set_columns(std::move(mutable_full_columns)); @@ -484,7 +506,6 @@ Status FlexibleReadPlan::read_columns_by_plan( const std::map& rsid_to_rowset, vectorized::Block& old_value_block, std::map* read_index) const { DCHECK(use_row_store); - auto mutable_columns = old_value_block.mutate_columns(); size_t read_idx = 0; for (const auto& [rowset_id, segment_row_mappings] : row_store_plan) { for (const auto& [segment_id, mappings] : segment_row_mappings) { @@ -495,19 +516,15 @@ Status FlexibleReadPlan::read_columns_by_plan( rids.emplace_back(rid); (*read_index)[pos] = read_idx++; } - for (size_t cid = 0; cid < mutable_columns.size(); ++cid) { - TabletColumn tablet_column = tablet_schema.column(cids_to_read[cid]); - auto st = doris::BaseTablet::fetch_value_by_rowids( - rowset_iter->second, segment_id, rids, tablet_column, mutable_columns[cid]); - // set read value to output block - if (!st.ok()) { - LOG(WARNING) << "failed to fetch value"; - return st; - } + auto st = BaseTablet::fetch_value_through_row_column(rowset_iter->second, tablet_schema, + segment_id, rids, cids_to_read, + old_value_block); + if (!st.ok()) { + LOG(WARNING) << "failed to fetch value through row column"; + return st; } } } - old_value_block.set_columns(std::move(mutable_columns)); return Status::OK(); } @@ -546,6 +563,11 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_column_store( const std::vector& use_default_or_null_flag, bool has_default_or_nullable, const std::size_t segment_start_pos, const std::size_t block_start_pos, const vectorized::Block* block, std::vector* skip_bitmaps) const { + auto* info = rowset_ctx->partial_update_info.get(); + int32_t seq_col_unique_id = -1; + if (tablet_schema.has_sequence_col()) { + seq_col_unique_id = tablet_schema.column(tablet_schema.sequence_col_idx()).unique_id(); + } // cid -> segment pos to write -> rowid to read in old_value_block std::map> read_index; RETURN_IF_ERROR( @@ -557,32 +579,61 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_column_store( auto default_value_block = old_value_block.clone_empty(); if (has_default_or_nullable || delete_sign_column_data != nullptr) { RETURN_IF_ERROR(BaseTablet::generate_default_value_block( - tablet_schema, non_sort_key_cids, rowset_ctx->partial_update_info->default_values, - old_value_block, default_value_block)); + tablet_schema, non_sort_key_cids, info->default_values, old_value_block, + default_value_block)); } - auto fill_one_cell = [&tablet_schema, &read_index]( + auto fill_one_cell = [&tablet_schema, &read_index, info]( const TabletColumn& tablet_column, uint32_t cid, vectorized::MutableColumnPtr& new_col, const vectorized::IColumn& default_value_col, const vectorized::IColumn& old_value_col, const vectorized::IColumn& cur_col, std::size_t block_pos, - std::size_t segment_pos, bool skipped, bool use_default, - const signed char* delete_sign_column_data) { + std::size_t segment_pos, bool skipped, bool row_has_sequence_col, + bool use_default, const signed char* delete_sign_column_data) { if (skipped) { DCHECK(cid != tablet_schema.skip_bitmap_col_idx()); DCHECK(cid != tablet_schema.version_col_idx()); DCHECK(!tablet_column.is_row_store_column()); - auto delete_sign_pos = read_index[tablet_schema.delete_sign_idx()][segment_pos]; - if (use_default || (delete_sign_column_data != nullptr && - delete_sign_column_data[delete_sign_pos] != 0)) { + if (!use_default) { + if (delete_sign_column_data != nullptr) { + bool old_row_delete_sign = false; + if (auto it = read_index[tablet_schema.delete_sign_idx()].find(segment_pos); + it != read_index[tablet_schema.delete_sign_idx()].end()) { + old_row_delete_sign = (delete_sign_column_data[it->second] != 0); + } + + if (old_row_delete_sign) { + if (!tablet_schema.has_sequence_col()) { + use_default = true; + } else if (row_has_sequence_col || + (!tablet_column.is_seqeunce_col() && + (tablet_column.unique_id() != info->sequence_map_col_uid()))) { + // to keep the sequence column value not decreasing, we should read values of seq column(and seq map column) + // from old rows even if the old row is deleted when the input don't specify the sequence column, otherwise + // it may cause the merge-on-read based compaction to produce incorrect results + use_default = true; + } + } + } + } + if (!use_default && tablet_column.is_on_update_current_timestamp()) { + use_default = true; + } + if (use_default) { if (tablet_column.has_default_value()) { new_col->insert_from(default_value_col, 0); } else if (tablet_column.is_nullable()) { assert_cast( new_col.get()) - ->insert_default(); + ->insert_many_defaults(1); + } else if (tablet_column.is_auto_increment()) { + // In flexible partial update, the skip bitmap indicates whether a cell + // is specified in the original load, so the generated auto-increment value is filled + // in current block in place if needed rather than using a seperate column to + // store the generated auto-increment value in fixed partial update + new_col->insert_from(cur_col, block_pos); } else { new_col->insert(tablet_column.get_vec_type()->get_default()); } @@ -609,6 +660,9 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_column_store( *old_value_block.get_by_position(i).column, *block->get_by_position(cid).column, block_pos, segment_pos, skip_bitmaps->at(block_pos).contains(col_uid), + tablet_schema.has_sequence_col() + ? !skip_bitmaps->at(block_pos).contains(seq_col_unique_id) + : false, use_default_or_null_flag[idx], delete_sign_column_data); } } @@ -622,6 +676,11 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_row_store( const std::vector& use_default_or_null_flag, bool has_default_or_nullable, const std::size_t segment_start_pos, const std::size_t block_start_pos, const vectorized::Block* block, std::vector* skip_bitmaps) const { + auto* info = rowset_ctx->partial_update_info.get(); + int32_t seq_col_unique_id = -1; + if (tablet_schema.has_sequence_col()) { + seq_col_unique_id = tablet_schema.column(tablet_schema.sequence_col_idx()).unique_id(); + } // segment pos to write -> rowid to read in old_value_block std::map read_index; RETURN_IF_ERROR(read_columns_by_plan(tablet_schema, non_sort_key_cids, rsid_to_rowset, @@ -632,31 +691,57 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_row_store( auto default_value_block = old_value_block.clone_empty(); if (has_default_or_nullable || delete_sign_column_data != nullptr) { RETURN_IF_ERROR(BaseTablet::generate_default_value_block( - tablet_schema, non_sort_key_cids, rowset_ctx->partial_update_info->default_values, - old_value_block, default_value_block)); - } - - auto fill_one_cell = [&tablet_schema](const TabletColumn& tablet_column, uint32_t cid, - vectorized::MutableColumnPtr& new_col, - const vectorized::IColumn& default_value_col, - const vectorized::IColumn& old_value_col, - const vectorized::IColumn& cur_col, std::size_t block_pos, - bool skipped, bool use_default, - const signed char* delete_sign_column_data, - uint32_t pos_in_old_block) { + tablet_schema, non_sort_key_cids, info->default_values, old_value_block, + default_value_block)); + } + + auto fill_one_cell = [&tablet_schema, info](const TabletColumn& tablet_column, uint32_t cid, + vectorized::MutableColumnPtr& new_col, + const vectorized::IColumn& default_value_col, + const vectorized::IColumn& old_value_col, + const vectorized::IColumn& cur_col, + std::size_t block_pos, bool skipped, + bool row_has_sequence_col, bool use_default, + const signed char* delete_sign_column_data, + uint32_t pos_in_old_block) { if (skipped) { DCHECK(cid != tablet_schema.skip_bitmap_col_idx()); DCHECK(cid != tablet_schema.version_col_idx()); DCHECK(!tablet_column.is_row_store_column()); + if (!use_default) { + if (delete_sign_column_data != nullptr) { + bool old_row_delete_sign = (delete_sign_column_data[pos_in_old_block] != 0); + if (old_row_delete_sign) { + if (!tablet_schema.has_sequence_col()) { + use_default = true; + } else if (row_has_sequence_col || + (!tablet_column.is_seqeunce_col() && + (tablet_column.unique_id() != info->sequence_map_col_uid()))) { + // to keep the sequence column value not decreasing, we should read values of seq column(and seq map column) + // from old rows even if the old row is deleted when the input don't specify the sequence column, otherwise + // it may cause the merge-on-read based compaction to produce incorrect results + use_default = true; + } + } + } + } - if (use_default || (delete_sign_column_data != nullptr && - delete_sign_column_data[pos_in_old_block] != 0)) { + if (!use_default && tablet_column.is_on_update_current_timestamp()) { + use_default = true; + } + if (use_default) { if (tablet_column.has_default_value()) { new_col->insert_from(default_value_col, 0); } else if (tablet_column.is_nullable()) { assert_cast( new_col.get()) - ->insert_default(); + ->insert_many_defaults(1); + } else if (tablet_column.is_auto_increment()) { + // In flexible partial update, the skip bitmap indicates whether a cell + // is specified in the original load, so the generated auto-increment value is filled + // in current block in place if needed rather than using a seperate column to + // store the generated auto-increment value in fixed partial update + new_col->insert_from(cur_col, block_pos); } else { new_col->insert(tablet_column.get_vec_type()->get_default()); } @@ -683,10 +768,421 @@ Status FlexibleReadPlan::fill_non_primary_key_columns_for_row_store( *old_value_block.get_by_position(i).column, *block->get_by_position(cid).column, block_pos, skip_bitmaps->at(block_pos).contains(col_uid), + tablet_schema.has_sequence_col() + ? !skip_bitmaps->at(block_pos).contains(seq_col_unique_id) + : false, use_default_or_null_flag[idx], delete_sign_column_data, pos_in_old_block); } } return Status::OK(); } +BlockAggregator::BlockAggregator(segment_v2::VerticalSegmentWriter& vertical_segment_writer) + : _writer(vertical_segment_writer), _tablet_schema(*_writer._tablet_schema) {} + +void BlockAggregator::merge_one_row(vectorized::MutableBlock& dst_block, + vectorized::Block* src_block, int64_t rid, + BitmapValue& skip_bitmap) { + for (size_t cid {_tablet_schema.num_key_columns()}; cid < _tablet_schema.num_columns(); cid++) { + if (cid == _tablet_schema.skip_bitmap_col_idx()) { + auto& cur_skip_bitmap = + assert_cast(dst_block.mutable_columns()[cid].get()) + ->get_data() + .back(); + const auto& new_row_skip_bitmap = + assert_cast( + src_block->get_by_position(cid).column->assume_mutable().get()) + ->get_data()[rid]; + cur_skip_bitmap &= new_row_skip_bitmap; + continue; + } + if (!skip_bitmap.contains(_tablet_schema.column(cid).unique_id())) { + dst_block.mutable_columns()[cid]->pop_back(1); + dst_block.mutable_columns()[cid]->insert_from(*src_block->get_by_position(cid).column, + rid); + } + } + VLOG_DEBUG << fmt::format("merge a row, after merge, output_block.rows()={}, state: {}", + dst_block.rows(), _state.to_string()); +} + +void BlockAggregator::append_one_row(vectorized::MutableBlock& dst_block, + vectorized::Block* src_block, int64_t rid) { + dst_block.add_row(src_block, rid); + _state.rows++; + VLOG_DEBUG << fmt::format("append a new row, after append, output_block.rows()={}, state: {}", + dst_block.rows(), _state.to_string()); +} + +void BlockAggregator::remove_last_n_rows(vectorized::MutableBlock& dst_block, int n) { + if (n > 0) { + for (size_t cid {0}; cid < _tablet_schema.num_columns(); cid++) { + DCHECK_GE(dst_block.mutable_columns()[cid]->size(), n); + dst_block.mutable_columns()[cid]->pop_back(n); + } + } +} + +void BlockAggregator::append_or_merge_row(vectorized::MutableBlock& dst_block, + vectorized::Block* src_block, int64_t rid, + BitmapValue& skip_bitmap, bool have_delete_sign) { + if (have_delete_sign) { + // remove all the previous batched rows + remove_last_n_rows(dst_block, _state.rows); + _state.rows = 0; + _state.has_row_with_delete_sign = true; + + append_one_row(dst_block, src_block, rid); + } else { + if (_state.should_merge()) { + merge_one_row(dst_block, src_block, rid, skip_bitmap); + } else { + append_one_row(dst_block, src_block, rid); + } + } +}; + +Status BlockAggregator::aggregate_rows( + vectorized::MutableBlock& output_block, vectorized::Block* block, int64_t start, + int64_t end, std::string key, std::vector* skip_bitmaps, + const signed char* delete_signs, vectorized::IOlapColumnDataAccessor* seq_column, + const std::vector& specified_rowsets, + std::vector>& segment_caches) { + VLOG_DEBUG << fmt::format("merge rows in range=[{}-{})", start, end); + if (end - start == 1) { + output_block.add_row(block, start); + VLOG_DEBUG << fmt::format("append a row directly, rid={}", start); + return Status::OK(); + } + + auto seq_col_unique_id = _tablet_schema.column(_tablet_schema.sequence_col_idx()).unique_id(); + auto delete_sign_col_unique_id = + _tablet_schema.column(_tablet_schema.delete_sign_idx()).unique_id(); + + _state.reset(); + + RowLocation loc; + RowsetSharedPtr rowset; + std::string previous_encoded_seq_value {}; + Status st = _writer._tablet->lookup_row_key( + key, &_tablet_schema, false, specified_rowsets, &loc, _writer._mow_context->max_version, + segment_caches, &rowset, true, &previous_encoded_seq_value); + int64_t pos = start; + bool is_expected_st = (st.is() || st.ok()); + DCHECK(is_expected_st || st.is()) + << "[BlockAggregator::aggregate_rows] unexpected error status while lookup_row_key:" + << st; + if (!is_expected_st) { + return st; + } + + std::string cur_seq_val; + if (st.ok()) { + for (pos = start; pos < end; pos++) { + auto& skip_bitmap = skip_bitmaps->at(pos); + bool row_has_sequence_col = (!skip_bitmap.contains(seq_col_unique_id)); + // Discard all the rows whose seq value is smaller than previous_encoded_seq_value. + if (row_has_sequence_col) { + std::string seq_val {}; + _writer._encode_seq_column(seq_column, pos, &seq_val); + if (Slice {seq_val}.compare(Slice {previous_encoded_seq_value}) < 0) { + continue; + } + cur_seq_val = std::move(seq_val); + break; + } + cur_seq_val = std::move(previous_encoded_seq_value); + break; + } + } else { + pos = start; + auto& skip_bitmap = skip_bitmaps->at(pos); + bool row_has_sequence_col = (!skip_bitmap.contains(seq_col_unique_id)); + if (row_has_sequence_col) { + std::string seq_val {}; + // for rows that don't specify seqeunce col, seq_val will be encoded to minial value + _writer._encode_seq_column(seq_column, pos, &seq_val); + cur_seq_val = std::move(seq_val); + } else { + cur_seq_val.clear(); + RETURN_IF_ERROR(_writer._generate_encoded_default_seq_value( + _tablet_schema, *_writer._opts.rowset_ctx->partial_update_info, &cur_seq_val)); + } + } + + for (int64_t rid {pos}; rid < end; rid++) { + auto& skip_bitmap = skip_bitmaps->at(rid); + bool row_has_sequence_col = (!skip_bitmap.contains(seq_col_unique_id)); + bool have_delete_sign = + (!skip_bitmap.contains(delete_sign_col_unique_id) && delete_signs[rid] != 0); + if (!row_has_sequence_col) { + append_or_merge_row(output_block, block, rid, skip_bitmap, have_delete_sign); + } else { + std::string seq_val {}; + _writer._encode_seq_column(seq_column, rid, &seq_val); + if (Slice {seq_val}.compare(Slice {cur_seq_val}) >= 0) { + append_or_merge_row(output_block, block, rid, skip_bitmap, have_delete_sign); + cur_seq_val = std::move(seq_val); + } else { + VLOG_DEBUG << fmt::format( + "skip rid={} becasue its seq value is lower than the previous", rid); + } + } + } + return Status::OK(); +}; + +Status BlockAggregator::aggregate_for_sequence_column( + vectorized::Block* block, size_t num_rows, + const std::vector& key_columns, + vectorized::IOlapColumnDataAccessor* seq_column, + const std::vector& specified_rowsets, + std::vector>& segment_caches) { + DCHECK_EQ(block->columns(), _tablet_schema.num_columns()); + // the process logic here is the same as MemTable::_aggregate_for_flexible_partial_update_without_seq_col() + // after this function, there will be at most 2 rows for a specified key + std::vector* skip_bitmaps = + &(assert_cast( + block->get_by_position(_tablet_schema.skip_bitmap_col_idx()) + .column->assume_mutable() + .get()) + ->get_data()); + const auto* delete_signs = BaseTablet::get_delete_sign_column_data(*block, num_rows); + + auto filtered_block = _tablet_schema.create_block(); + vectorized::MutableBlock output_block = + vectorized::MutableBlock::build_mutable_block(&filtered_block); + + int same_key_rows {0}; + std::string previous_key {}; + for (size_t block_pos {0}; block_pos < num_rows; block_pos++) { + std::string key = _writer._full_encode_keys(key_columns, block_pos); + if (block_pos > 0 && previous_key == key) { + same_key_rows++; + } else { + if (same_key_rows > 0) { + RETURN_IF_ERROR(aggregate_rows(output_block, block, block_pos - same_key_rows, + block_pos, std::move(previous_key), skip_bitmaps, + delete_signs, seq_column, specified_rowsets, + segment_caches)); + } + same_key_rows = 1; + } + previous_key = std::move(key); + } + if (same_key_rows > 0) { + RETURN_IF_ERROR(aggregate_rows(output_block, block, num_rows - same_key_rows, num_rows, + std::move(previous_key), skip_bitmaps, delete_signs, + seq_column, specified_rowsets, segment_caches)); + } + + block->swap(output_block.to_block()); + return Status::OK(); +} + +Status BlockAggregator::fill_sequence_column(vectorized::Block* block, size_t num_rows, + const FixedReadPlan& read_plan, + std::vector& skip_bitmaps) { + DCHECK(_tablet_schema.has_sequence_col()); + std::vector cids {static_cast(_tablet_schema.sequence_col_idx())}; + auto seq_col_unique_id = _tablet_schema.column(_tablet_schema.sequence_col_idx()).unique_id(); + + auto seq_col_block = _tablet_schema.create_block_by_cids(cids); + auto tmp_block = _tablet_schema.create_block_by_cids(cids); + std::map read_index; + RETURN_IF_ERROR(read_plan.read_columns_by_plan(_tablet_schema, cids, _writer._rsid_to_rowset, + seq_col_block, &read_index, false)); + + auto new_seq_col_ptr = tmp_block.get_by_position(0).column->assume_mutable(); + const auto& old_seq_col_ptr = *seq_col_block.get_by_position(0).column; + const auto& cur_seq_col_ptr = *block->get_by_position(_tablet_schema.sequence_col_idx()).column; + for (size_t block_pos {0}; block_pos < num_rows; block_pos++) { + if (read_index.contains(block_pos)) { + new_seq_col_ptr->insert_from(old_seq_col_ptr, read_index[block_pos]); + skip_bitmaps[block_pos].remove(seq_col_unique_id); + } else { + new_seq_col_ptr->insert_from(cur_seq_col_ptr, block_pos); + } + } + block->replace_by_position(_tablet_schema.sequence_col_idx(), std::move(new_seq_col_ptr)); + return Status::OK(); +} + +Status BlockAggregator::aggregate_for_insert_after_delete( + vectorized::Block* block, size_t num_rows, + const std::vector& key_columns, + const std::vector& specified_rowsets, + std::vector>& segment_caches) { + DCHECK_EQ(block->columns(), _tablet_schema.num_columns()); + // there will be at most 2 rows for a specified key in block when control flow reaches here + // after this function, there will not be duplicate rows in block + + std::vector* skip_bitmaps = + &(assert_cast( + block->get_by_position(_tablet_schema.skip_bitmap_col_idx()) + .column->assume_mutable() + .get()) + ->get_data()); + const auto* delete_signs = BaseTablet::get_delete_sign_column_data(*block, num_rows); + + auto filter_column = vectorized::ColumnUInt8::create(num_rows, 1); + auto* __restrict filter_map = filter_column->get_data().data(); + std::string previous_key {}; + bool previous_has_delete_sign {false}; + int duplicate_rows {0}; + int32_t delete_sign_col_unique_id = + _tablet_schema.column(_tablet_schema.delete_sign_idx()).unique_id(); + auto seq_col_unique_id = + (_tablet_schema.sequence_col_idx() != -1) + ? _tablet_schema.column(_tablet_schema.sequence_col_idx()).unique_id() + : -1; + FixedReadPlan read_plan; + for (size_t block_pos {0}; block_pos < num_rows; block_pos++) { + size_t delta_pos = block_pos; + auto& skip_bitmap = skip_bitmaps->at(block_pos); + std::string key = _writer._full_encode_keys(key_columns, delta_pos); + bool have_delete_sign = + (!skip_bitmap.contains(delete_sign_col_unique_id) && delete_signs[block_pos] != 0); + if (delta_pos > 0 && previous_key == key) { + // !!ATTENTION!!: We can only remove the row with delete sign if there is a insert with the same key after this row. + // If there is only a row with delete sign, we should keep it and can't remove it from block, because + // compaction will not use the delete bitmap when reading data. So there may still be rows with delete sign + // in later process + DCHECK(previous_has_delete_sign); + DCHECK(!have_delete_sign); + ++duplicate_rows; + RowLocation loc; + RowsetSharedPtr rowset; + Status st = _writer._tablet->lookup_row_key( + key, &_tablet_schema, false, specified_rowsets, &loc, + _writer._mow_context->max_version, segment_caches, &rowset, true); + bool is_expected_st = (st.is() || st.ok()); + DCHECK(is_expected_st || st.is()) + << "[BlockAggregator::aggregate_for_insert_after_delete] unexpected error " + "status while lookup_row_key:" + << st; + if (!is_expected_st) { + return st; + } + + Slice previous_seq_slice {}; + if (st.ok()) { + if (_tablet_schema.has_sequence_col()) { + // if the insert row doesn't specify the sequence column, we need to + // read the historical's sequence column value so that we don't need + // to handle seqeunce column in append_block_with_flexible_content() + // for this row + bool row_has_sequence_col = (!skip_bitmap.contains(seq_col_unique_id)); + if (!row_has_sequence_col) { + read_plan.prepare_to_read(loc, block_pos); + _writer._rsid_to_rowset.emplace(rowset->rowset_id(), rowset); + } + } + // delete the existing row + _writer._mow_context->delete_bitmap->add( + {loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, + loc.row_id); + } + // and remove the row with delete sign from the current block + filter_map[block_pos - 1] = 0; + } + previous_has_delete_sign = have_delete_sign; + previous_key = std::move(key); + } + if (duplicate_rows > 0) { + if (!read_plan.empty()) { + // fill sequence column value for some rows + RETURN_IF_ERROR(fill_sequence_column(block, num_rows, read_plan, *skip_bitmaps)); + } + RETURN_IF_ERROR(filter_block(block, num_rows, std::move(filter_column), duplicate_rows, + "__filter_insert_after_delete_col__")); + } + return Status::OK(); +} + +Status BlockAggregator::filter_block(vectorized::Block* block, size_t num_rows, + vectorized::MutableColumnPtr filter_column, int duplicate_rows, + std::string col_name) { + auto num_cols = block->columns(); + block->insert( + {std::move(filter_column), std::make_shared(), col_name}); + RETURN_IF_ERROR(vectorized::Block::filter_block(block, num_cols, num_cols)); + DCHECK_EQ(num_cols, block->columns()); + int merged_rows = num_rows - block->rows(); + if (duplicate_rows != merged_rows) { + auto msg = fmt::format( + "filter_block_for_flexible_partial_update {}: duplicate_rows != merged_rows, " + "duplicate_keys={}, merged_rows={}, num_rows={}, mutable_block->rows()={}", + col_name, duplicate_rows, merged_rows, num_rows, block->rows()); + DCHECK(false) << msg; + return Status::InternalError(msg); + } + return Status::OK(); +} + +Status BlockAggregator::convert_pk_columns( + vectorized::Block* block, size_t row_pos, size_t num_rows, + std::vector& key_columns) { + key_columns.clear(); + for (std::size_t cid {0}; cid < _tablet_schema.num_key_columns(); cid++) { + RETURN_IF_ERROR(_writer._olap_data_convertor->set_source_content_with_specifid_column( + block->get_by_position(cid), row_pos, num_rows, cid)); + auto [status, column] = _writer._olap_data_convertor->convert_column_data(cid); + if (!status.ok()) { + return status; + } + key_columns.push_back(column); + } + return Status::OK(); +} + +Status BlockAggregator::convert_seq_column(vectorized::Block* block, size_t row_pos, + size_t num_rows, + vectorized::IOlapColumnDataAccessor*& seq_column) { + seq_column = nullptr; + if (_tablet_schema.has_sequence_col()) { + auto seq_col_idx = _tablet_schema.sequence_col_idx(); + RETURN_IF_ERROR(_writer._olap_data_convertor->set_source_content_with_specifid_column( + block->get_by_position(seq_col_idx), row_pos, num_rows, seq_col_idx)); + auto [status, column] = _writer._olap_data_convertor->convert_column_data(seq_col_idx); + if (!status.ok()) { + return status; + } + seq_column = column; + } + return Status::OK(); +}; + +Status BlockAggregator::aggregate_for_flexible_partial_update( + vectorized::Block* block, size_t num_rows, + const std::vector& specified_rowsets, + std::vector>& segment_caches) { + std::vector key_columns {}; + vectorized::IOlapColumnDataAccessor* seq_column {nullptr}; + + RETURN_IF_ERROR(convert_pk_columns(block, 0, num_rows, key_columns)); + RETURN_IF_ERROR(convert_seq_column(block, 0, num_rows, seq_column)); + + // 1. merge duplicate rows when table has sequence column + // When there are multiple rows with the same keys in memtable, some of them specify specify the sequence column, + // some of them don't. We can't do the de-duplication in memtable because we don't know the historical data. We must + // de-duplicate them here. + if (_tablet_schema.has_sequence_col()) { + RETURN_IF_ERROR(aggregate_for_sequence_column(block, num_rows, key_columns, seq_column, + specified_rowsets, segment_caches)); + } + + // 2. merge duplicate rows and handle insert after delete + if (block->rows() != num_rows) { + num_rows = block->rows(); + // data in block has changed, should re-encode key columns, sequence column + _writer._olap_data_convertor->clear_source_content(); + RETURN_IF_ERROR(convert_pk_columns(block, 0, num_rows, key_columns)); + RETURN_IF_ERROR(convert_seq_column(block, 0, num_rows, seq_column)); + } + RETURN_IF_ERROR(aggregate_for_insert_after_delete(block, num_rows, key_columns, + specified_rowsets, segment_caches)); + return Status::OK(); +} + } // namespace doris diff --git a/be/src/olap/partial_update_info.h b/be/src/olap/partial_update_info.h index 806db586f5137e..fbf3dad71ad3cf 100644 --- a/be/src/olap/partial_update_info.h +++ b/be/src/olap/partial_update_info.h @@ -37,10 +37,18 @@ class BitmapValue; struct RowLocation; namespace vectorized { class Block; -} +class MutableBlock; +class IOlapColumnDataAccessor; + +} // namespace vectorized struct RowsetWriterContext; struct RowsetId; class BitmapValue; +namespace segment_v2 { +class VerticalSegmentWriter; +} + +class SegmentCacheHandle; struct PartialUpdateInfo { Status init(int64_t tablet_id, int64_t txn_id, const TabletSchema& tablet_schema, @@ -112,6 +120,7 @@ struct RidAndPos { class FixedReadPlan { public: + bool empty() const; void prepare_to_read(const RowLocation& row_location, size_t pos); Status read_columns_by_plan(const TabletSchema& tablet_schema, std::vector cids_to_read, @@ -179,6 +188,79 @@ class FlexibleReadPlan { std::map>> row_store_plan; }; +class BlockAggregator { +public: + ~BlockAggregator() = default; + BlockAggregator(segment_v2::VerticalSegmentWriter& vertical_segment_writer); + + Status convert_pk_columns(vectorized::Block* block, size_t row_pos, size_t num_rows, + std::vector& key_columns); + Status convert_seq_column(vectorized::Block* block, size_t row_pos, size_t num_rows, + vectorized::IOlapColumnDataAccessor*& seq_column); + Status aggregate_for_flexible_partial_update( + vectorized::Block* block, size_t num_rows, + const std::vector& specified_rowsets, + std::vector>& segment_caches); + +private: + Status aggregate_for_sequence_column( + vectorized::Block* block, size_t num_rows, + const std::vector& key_columns, + vectorized::IOlapColumnDataAccessor* seq_column, + const std::vector& specified_rowsets, + std::vector>& segment_caches); + Status aggregate_for_insert_after_delete( + vectorized::Block* block, size_t num_rows, + const std::vector& key_columns, + const std::vector& specified_rowsets, + std::vector>& segment_caches); + Status filter_block(vectorized::Block* block, size_t num_rows, + vectorized::MutableColumnPtr filter_column, int duplicate_rows, + std::string col_name); + + Status fill_sequence_column(vectorized::Block* block, size_t num_rows, + const FixedReadPlan& read_plan, + std::vector& skip_bitmaps); + + void append_or_merge_row(vectorized::MutableBlock& dst_block, vectorized::Block* src_block, + int64_t rid, BitmapValue& skip_bitmap, bool have_delete_sign); + void merge_one_row(vectorized::MutableBlock& dst_block, vectorized::Block* src_block, + int64_t rid, BitmapValue& skip_bitmap); + void append_one_row(vectorized::MutableBlock& dst_block, vectorized::Block* src_block, + int64_t rid); + void remove_last_n_rows(vectorized::MutableBlock& dst_block, int n); + + // aggregate rows with same keys in range [start, end) from block to output_block + Status aggregate_rows(vectorized::MutableBlock& output_block, vectorized::Block* block, + int64_t start, int64_t end, std::string key, + std::vector* skip_bitmaps, const signed char* delete_signs, + vectorized::IOlapColumnDataAccessor* seq_column, + const std::vector& specified_rowsets, + std::vector>& segment_caches); + + segment_v2::VerticalSegmentWriter& _writer; + TabletSchema& _tablet_schema; + + // used to store state when aggregating rows in block + struct AggregateState { + int rows {0}; + bool has_row_with_delete_sign {false}; + + bool should_merge() const { + return ((rows == 1 && !has_row_with_delete_sign) || rows == 2); + } + + void reset() { + rows = 0; + has_row_with_delete_sign = false; + } + + std::string to_string() const { + return fmt::format("rows={}, have_delete_row={}", rows, has_row_with_delete_sign); + } + } _state {}; +}; + struct PartialUpdateStats { int64_t num_rows_updated {0}; int64_t num_rows_new_added {0}; diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 4e442975cc655c..99a39c61d20ec1 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -517,7 +517,11 @@ Status SegmentWriter::probe_key_for_mow( // 2. the one exception is when there are sequence columns in the table, we need to read // the sequence columns, otherwise it may cause the merge-on-read based compaction // policy to produce incorrect results - if (have_delete_sign && !_tablet_schema->has_sequence_col()) { + // TODO(bobhan1): only read seq col rather than all columns in this situation for + // partial update and flexible partial update + + // TODO(bobhan1): handle sequence column here + if (st.is() || (have_delete_sign && !_tablet_schema->has_sequence_col())) { has_default_or_nullable = true; use_default_or_null_flag.emplace_back(true); } else { @@ -635,7 +639,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* bool has_default_or_nullable = false; std::vector use_default_or_null_flag; use_default_or_null_flag.reserve(num_rows); - const auto* delete_sign_column_data = + const auto* delete_signs = BaseTablet::get_delete_sign_column_data(full_block, row_pos + num_rows); const std::vector& specified_rowsets = _mow_context->rowset_ptrs; @@ -668,8 +672,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* } // mark key with delete sign as deleted. - bool have_delete_sign = - (delete_sign_column_data != nullptr && delete_sign_column_data[block_pos] != 0); + bool have_delete_sign = (delete_signs != nullptr && delete_signs[block_pos] != 0); auto not_found_cb = [&]() { return _opts.rowset_ctx->partial_update_info->handle_new_key( diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 9eab36bd2d2fc0..1b98095f774e5e 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -99,7 +99,8 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32 _index_file_writer(index_file_writer), _mem_tracker(std::make_unique( vertical_segment_writer_mem_tracker_name(segment_id))), - _mow_context(std::move(opts.mow_ctx)) { + _mow_context(std::move(opts.mow_ctx)), + _block_aggregator(*this) { CHECK_NOTNULL(file_writer); _num_sort_key_columns = _tablet_schema->num_key_columns(); _num_short_key_columns = _tablet_schema->num_short_key_columns(); @@ -393,7 +394,15 @@ Status VerticalSegmentWriter::_probe_key_for_mow( // 2. the one exception is when there are sequence columns in the table, we need to read // the sequence columns, otherwise it may cause the merge-on-read based compaction // policy to produce incorrect results - if (have_delete_sign && !_tablet_schema->has_sequence_col()) { + + // 3. In flexible partial update, we may delete the existing rows before if there exists + // insert after delete in one load. In this case, the insert should also be treated + // as newly inserted rows, note that the sequence column value is filled in + // BlockAggregator::aggregate_for_insert_after_delete() if this row doesn't specify the sequence column + if (st.is() || (have_delete_sign && !_tablet_schema->has_sequence_col()) || + (_opts.rowset_ctx->partial_update_info->is_flexible_partial_update() && + _mow_context->delete_bitmap->contains( + {loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, loc.row_id))) { has_default_or_nullable = true; use_default_or_null_flag.emplace_back(true); } else { @@ -512,7 +521,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da bool has_default_or_nullable = false; std::vector use_default_or_null_flag; use_default_or_null_flag.reserve(data.num_rows); - const auto* delete_sign_column_data = + const auto* delete_signs = BaseTablet::get_delete_sign_column_data(full_block, data.row_pos + data.num_rows); DBUG_EXECUTE_IF("VerticalSegmentWriter._append_block_with_partial_content.sleep", @@ -547,8 +556,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da } // mark key with delete sign as deleted. - bool have_delete_sign = - (delete_sign_column_data != nullptr && delete_sign_column_data[block_pos] != 0); + bool have_delete_sign = (delete_signs != nullptr && delete_signs[block_pos] != 0); auto not_found_cb = [&]() { return _opts.rowset_ctx->partial_update_info->handle_new_key( @@ -638,13 +646,6 @@ Status VerticalSegmentWriter::_append_block_with_flexible_partial_content( DCHECK(_tablet_schema->has_skip_bitmap_col()); auto skip_bitmap_col_idx = _tablet_schema->skip_bitmap_col_idx(); - auto get_skip_bitmaps = [&skip_bitmap_col_idx](const vectorized::Block* block) { - return &(assert_cast( - block->get_by_position(skip_bitmap_col_idx).column->assume_mutable().get()) - ->get_data()); - }; - std::vector* skip_bitmaps = get_skip_bitmaps(data.block); - bool has_default_or_nullable = false; std::vector use_default_or_null_flag; use_default_or_null_flag.reserve(data.num_rows); @@ -657,77 +658,44 @@ Status VerticalSegmentWriter::_append_block_with_flexible_partial_content( const std::vector& specified_rowsets = _mow_context->rowset_ptrs; std::vector> segment_caches(specified_rowsets.size()); - std::vector key_columns {}; - vectorized::IOlapColumnDataAccessor* seq_column {nullptr}; - - auto encode_key_columns = - [&full_block, &data, - this](std::vector& key_columns) -> Status { - key_columns.clear(); - for (std::size_t cid {0}; cid < _num_sort_key_columns; cid++) { - full_block.replace_by_position(cid, data.block->get_by_position(cid).column); - RETURN_IF_ERROR(_olap_data_convertor->set_source_content_with_specifid_column( - full_block.get_by_position(cid), data.row_pos, data.num_rows, cid)); - auto [status, column] = _olap_data_convertor->convert_column_data(cid); - if (!status.ok()) { - return status; - } - key_columns.push_back(column); - } - return Status::OK(); - }; - - auto encode_seq_column = [&data, &schema_has_sequence_col, - this](vectorized::IOlapColumnDataAccessor*& seq_column) -> Status { - seq_column = nullptr; - if (schema_has_sequence_col) { - auto seq_col_idx = _tablet_schema->sequence_col_idx(); - RETURN_IF_ERROR(_olap_data_convertor->set_source_content_with_specifid_column( - data.block->get_by_position(seq_col_idx), data.row_pos, data.num_rows, - seq_col_idx)); - auto [status, column] = _olap_data_convertor->convert_column_data(seq_col_idx); - if (!status.ok()) { - return status; - } - seq_column = column; - } - return Status::OK(); - }; + // 1. aggregate duplicate rows in block + RETURN_IF_ERROR(_block_aggregator.aggregate_for_flexible_partial_update( + const_cast(data.block), data.num_rows, specified_rowsets, + segment_caches)); + if (data.block->rows() != data.num_rows) { + data.num_rows = data.block->rows(); + _olap_data_convertor->clear_source_content(); + } - // 1. encode key columns - // we can only encode sort key columns currently becasue all non-key columns in flexible partial update + // 2. encode primary key columns + // we can only encode primary key columns currently becasue all non-primary columns in flexible partial update // can have missing cells - RETURN_IF_ERROR(encode_key_columns(key_columns)); - - // 2. encode sequence column + std::vector key_columns {}; + RETURN_IF_ERROR(_block_aggregator.convert_pk_columns(const_cast(data.block), + data.row_pos, data.num_rows, key_columns)); + // 3. encode sequence column // We encode the seguence column even thought it may have invalid values in some rows because we need to // encode the value of sequence column in key for rows that have a valid value in sequence column during // lookup_raw_key. We will encode the sequence column again at the end of this method. At that time, we have // a valid sequence column to encode the key with seq col. - RETURN_IF_ERROR(encode_seq_column(seq_column)); - - // 3. merge duplicate rows when table has sequence column - // When there are multiple rows with the same keys in memtable, some of them specify specify the sequence column, - // some of them don't. We can't do the de-duplication in memtable. We must de-duplicate them here. - if (schema_has_sequence_col) { - std::size_t origin_rows = data.num_rows; - RETURN_IF_ERROR(_merge_rows_for_sequence_column(data, skip_bitmaps, key_columns, seq_column, - specified_rowsets, segment_caches)); - if (origin_rows != data.num_rows) { - // data in block has changed, should re-encode key columns, sequence column and re-get skip_bitmaps - _olap_data_convertor->clear_source_content(); - RETURN_IF_ERROR(encode_key_columns(key_columns)); - RETURN_IF_ERROR(encode_seq_column(seq_column)); - skip_bitmaps = get_skip_bitmaps(data.block); - } - } - - const auto* delete_sign_column_data = + vectorized::IOlapColumnDataAccessor* seq_column {nullptr}; + RETURN_IF_ERROR(_block_aggregator.convert_seq_column(const_cast(data.block), + data.row_pos, data.num_rows, seq_column)); + + std::vector* skip_bitmaps = &( + assert_cast( + data.block->get_by_position(skip_bitmap_col_idx).column->assume_mutable().get()) + ->get_data()); + const auto* delete_signs = BaseTablet::get_delete_sign_column_data(*data.block, data.row_pos + data.num_rows); - DCHECK(delete_sign_column_data != nullptr); + DCHECK(delete_signs != nullptr); + + for (std::size_t cid {0}; cid < _tablet_schema->num_key_columns(); cid++) { + full_block.replace_by_position(cid, data.block->get_by_position(cid).column); + } - // 4. write key columns data - for (std::size_t cid {0}; cid < _num_sort_key_columns; cid++) { + // 4. write primary key columns data + for (std::size_t cid {0}; cid < _tablet_schema->num_key_columns(); cid++) { const auto& column = key_columns[cid]; DCHECK(_column_writers[cid]->get_next_rowid() == _num_rows_written); RETURN_IF_ERROR(_column_writers[cid]->append(column->get_nullmap(), column->get_data(), @@ -740,8 +708,8 @@ Status VerticalSegmentWriter::_append_block_with_flexible_partial_content( PartialUpdateStats stats; RETURN_IF_ERROR(_generate_flexible_read_plan( read_plan, data, segment_start_pos, schema_has_sequence_col, seq_map_col_unique_id, - skip_bitmaps, key_columns, seq_column, delete_sign_column_data, specified_rowsets, - segment_caches, has_default_or_nullable, use_default_or_null_flag, stats)); + skip_bitmaps, key_columns, seq_column, delete_signs, specified_rowsets, segment_caches, + has_default_or_nullable, use_default_or_null_flag, stats)); CHECK_EQ(use_default_or_null_flag.size(), data.num_rows); if (config::enable_merge_on_write_correctness_check) { @@ -762,7 +730,7 @@ Status VerticalSegmentWriter::_append_block_with_flexible_partial_content( _serialize_block_to_row_column(full_block); // 8. encode and write all non-primary key columns(including sequence column if exists) - for (auto cid = _num_sort_key_columns; cid < _tablet_schema->num_columns(); cid++) { + for (auto cid = _tablet_schema->num_key_columns(); cid < _tablet_schema->num_columns(); cid++) { RETURN_IF_ERROR(_olap_data_convertor->set_source_content_with_specifid_column( full_block.get_by_position(cid), data.row_pos, data.num_rows, cid)); auto [status, column] = _olap_data_convertor->convert_column_data(cid); @@ -838,7 +806,7 @@ Status VerticalSegmentWriter::_generate_flexible_read_plan( bool schema_has_sequence_col, int32_t seq_map_col_unique_id, std::vector* skip_bitmaps, const std::vector& key_columns, - vectorized::IOlapColumnDataAccessor* seq_column, const signed char* delete_sign_column_data, + vectorized::IOlapColumnDataAccessor* seq_column, const signed char* delete_signs, const std::vector& specified_rowsets, std::vector>& segment_caches, bool& has_default_or_nullable, std::vector& use_default_or_null_flag, @@ -854,14 +822,6 @@ Status VerticalSegmentWriter::_generate_flexible_read_plan( size_t segment_pos = segment_start_pos + delta_pos; auto& skip_bitmap = skip_bitmaps->at(block_pos); - // the hidden sequence column should have the same mark with sequence map column - if (seq_map_col_unique_id != -1) { - DCHECK(schema_has_sequence_col); - if (skip_bitmap.contains(seq_map_col_unique_id)) { - skip_bitmap.add(seq_col_unique_id); - } - } - std::string key = _full_encode_keys(key_columns, delta_pos); _maybe_invalid_row_cache(key); bool row_has_sequence_col = @@ -871,8 +831,8 @@ Status VerticalSegmentWriter::_generate_flexible_read_plan( } // mark key with delete sign as deleted. - bool have_delete_sign = (!skip_bitmap.contains(delete_sign_col_unique_id) && - delete_sign_column_data[block_pos] != 0); + bool have_delete_sign = + (!skip_bitmap.contains(delete_sign_col_unique_id) && delete_signs[block_pos] != 0); auto not_found_cb = [&]() { return _opts.rowset_ctx->partial_update_info->handle_new_key( @@ -894,104 +854,6 @@ Status VerticalSegmentWriter::_generate_flexible_read_plan( return Status::OK(); } -Status VerticalSegmentWriter::_merge_rows_for_sequence_column( - RowsInBlock& data, std::vector* skip_bitmaps, - const std::vector& key_columns, - vectorized::IOlapColumnDataAccessor* seq_column, - const std::vector& specified_rowsets, - std::vector>& segment_caches) { - VLOG_DEBUG << fmt::format( - "VerticalSegmentWriter::_merge_rows_for_sequence_column enter: data.block:{}\n", - data.block->dump_data()); - auto seq_col_unique_id = _tablet_schema->column(_tablet_schema->sequence_col_idx()).unique_id(); - std::string previous_key {}; - bool previous_has_seq_col {false}; - int duplicate_keys {0}; - - auto filter_column = vectorized::ColumnUInt8::create(data.num_rows, 1); - auto* __restrict filter_map = filter_column->get_data().data(); - - std::string encoded_default_seq_value {}; - RETURN_IF_ERROR(_generate_encoded_default_seq_value( - *_tablet_schema, *_opts.rowset_ctx->partial_update_info, &encoded_default_seq_value)); - - for (size_t block_pos = data.row_pos; block_pos < data.row_pos + data.num_rows; block_pos++) { - size_t delta_pos = block_pos - data.row_pos; - auto& skip_bitmap = skip_bitmaps->at(block_pos); - std::string key = _full_encode_keys(key_columns, delta_pos); - bool row_has_sequence_col = (!skip_bitmap.contains(seq_col_unique_id)); - Status st; - if (delta_pos > 0 && previous_key == key) { - DCHECK(previous_has_seq_col == !row_has_sequence_col); - ++duplicate_keys; - RowLocation loc; - RowsetSharedPtr rowset; - size_t rid_missing_seq {}; - size_t rid_with_seq {}; - if (row_has_sequence_col) { - rid_missing_seq = block_pos - 1; - rid_with_seq = block_pos; - } else { - rid_missing_seq = block_pos; - rid_with_seq = block_pos - 1; - } - std::string previous_encoded_seq_value {}; - st = _tablet->lookup_row_key(key, _tablet_schema.get(), false, specified_rowsets, &loc, - _mow_context->max_version, segment_caches, &rowset, true, - &previous_encoded_seq_value); - DCHECK(st.is() || st.ok()); - - Slice previous_seq_slice {}; - if (st.is()) { - previous_seq_slice = Slice {encoded_default_seq_value}; - } else { - // TODO(bobhan1): we can mark these rows in delete bitmap and eliminate reading them in later phase - _rsid_to_rowset.emplace(rowset->rowset_id(), rowset); - previous_seq_slice = Slice {previous_encoded_seq_value}; - } - std::string cur_encoded_seq_value {}; - _encode_seq_column(seq_column, rid_with_seq, &cur_encoded_seq_value); - // the encoded value is order-preserving, so we can use Slice::compare() to compare them - int res = previous_seq_slice.compare(Slice {cur_encoded_seq_value}); - VLOG_DEBUG << fmt::format( - "VerticalSegmentWriter::_merge_rows_for_sequence_column: rid_with_seq={}, " - "rid_missing_seq={}, res={}", - rid_with_seq, rid_missing_seq, res); - if (res > 0) { - filter_map[rid_with_seq] = 0; - } else if (res < 0) { - filter_map[rid_missing_seq] = 0; - } else { - filter_map[std::min(rid_with_seq, rid_missing_seq)] = 0; - } - } - previous_key = std::move(key); - previous_has_seq_col = row_has_sequence_col; - } - if (duplicate_keys > 0) { - auto num_cols = data.block->columns(); - auto* block = const_cast(data.block); - block->insert({std::move(filter_column), std::make_shared(), - "__dup_key_filter_col__"}); - RETURN_IF_ERROR(vectorized::Block::filter_block(block, num_cols, num_cols)); - int merged_rows = data.num_rows - block->rows(); - VLOG_DEBUG << fmt::format( - "VerticalSegmentWriter::_merge_rows_for_sequence_column after filter: " - "data.block:{}\n", - data.block->dump_data()); - if (duplicate_keys != merged_rows) { - auto msg = fmt::format( - "duplicate_keys != merged_rows, duplicate_keys={}, merged_rows={}, " - "num_rows={}, mutable_block->rows()={}", - duplicate_keys, merged_rows, data.num_rows, block->rows()); - DCHECK(false) << msg; - return Status::InternalError(msg); - } - data.num_rows = block->rows(); - } - return Status::OK(); -} - Status VerticalSegmentWriter::batch_block(const vectorized::Block* block, size_t row_pos, size_t num_rows) { if (_opts.rowset_ctx->partial_update_info && diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h index c5cd101b7849c1..e593b4009445a8 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h @@ -31,6 +31,7 @@ #include "common/status.h" // Status #include "olap/olap_define.h" +#include "olap/partial_update_info.h" #include "olap/rowset/segment_v2/column_writer.h" #include "olap/rowset/segment_v2/index_file_writer.h" #include "olap/tablet.h" @@ -56,7 +57,6 @@ namespace io { class FileWriter; class FileSystem; } // namespace io - namespace segment_v2 { class IndexFileWriter; @@ -185,18 +185,11 @@ class VerticalSegmentWriter { bool schema_has_sequence_col, int32_t seq_map_col_unique_id, std::vector* skip_bitmaps, const std::vector& key_columns, - vectorized::IOlapColumnDataAccessor* seq_column, - const signed char* delete_sign_column_data, + vectorized::IOlapColumnDataAccessor* seq_column, const signed char* delete_signs, const std::vector& specified_rowsets, std::vector>& segment_caches, bool& has_default_or_nullable, std::vector& use_default_or_null_flag, PartialUpdateStats& stats); - Status _merge_rows_for_sequence_column( - RowsInBlock& data, std::vector* skip_bitmaps, - const std::vector& key_columns, - vectorized::IOlapColumnDataAccessor* seq_column, - const std::vector& specified_rowsets, - std::vector>& segment_caches); Status _append_block_with_variant_subcolumns(RowsInBlock& data); Status _generate_key_index( RowsInBlock& data, std::vector& key_columns, @@ -212,6 +205,7 @@ class VerticalSegmentWriter { bool _is_mow_with_cluster_key(); private: + friend class ::doris::BlockAggregator; uint32_t _segment_id; TabletSchemaSPtr _tablet_schema; BaseTabletSPtr _tablet; @@ -271,6 +265,8 @@ class VerticalSegmentWriter { // contains auto generated columns, should be nullptr if no variants's subcolumns TabletSchemaSPtr _flush_schema = nullptr; + + BlockAggregator _block_aggregator; }; } // namespace segment_v2 diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index cd5072457725ee..f166144bb4e45c 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -401,6 +401,9 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco column->set_name(tcolumn.column_name); column->set_has_bitmap_index(tcolumn.has_bitmap_index); column->set_is_auto_increment(tcolumn.is_auto_increment); + if (tcolumn.__isset.is_on_update_current_timestamp) { + column->set_is_on_update_current_timestamp(tcolumn.is_on_update_current_timestamp); + } string data_type; EnumToString(TPrimitiveType, tcolumn.column_type.type, data_type); column->set_type(data_type); diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 8390c230865ff1..7f679376e34db5 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -526,6 +526,9 @@ void TabletColumn::init_from_pb(const ColumnPB& column) { _is_key = column.is_key(); _is_nullable = column.is_nullable(); _is_auto_increment = column.is_auto_increment(); + if (column.has_is_on_update_current_timestamp()) { + _is_on_update_current_timestamp = column.is_on_update_current_timestamp(); + } _has_default_value = column.has_default_value(); if (_has_default_value) { @@ -622,6 +625,8 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const { column->set_type(get_string_by_field_type(_type)); column->set_is_key(_is_key); column->set_is_nullable(_is_nullable); + column->set_is_auto_increment(_is_auto_increment); + column->set_is_on_update_current_timestamp(_is_on_update_current_timestamp); if (_has_default_value) { column->set_default_value(_default_value); } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 85feb62d51a7f1..559482f16d352e 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -88,6 +88,8 @@ class TabletColumn : public MetadataAdder { bool is_key() const { return _is_key; } bool is_nullable() const { return _is_nullable; } bool is_auto_increment() const { return _is_auto_increment; } + bool is_seqeunce_col() const { return _col_name == SEQUENCE_COL; } + bool is_on_update_current_timestamp() const { return _is_on_update_current_timestamp; } bool is_variant_type() const { return _type == FieldType::OLAP_FIELD_TYPE_VARIANT; } bool is_bf_column() const { return _is_bf_column; } bool has_bitmap_index() const { return _has_bitmap_index; } @@ -121,6 +123,9 @@ class TabletColumn : public MetadataAdder { void set_is_key(bool is_key) { _is_key = is_key; } void set_is_nullable(bool is_nullable) { _is_nullable = is_nullable; } void set_is_auto_increment(bool is_auto_increment) { _is_auto_increment = is_auto_increment; } + void set_is_on_update_current_timestamp(bool is_on_update_current_timestamp) { + _is_on_update_current_timestamp = is_on_update_current_timestamp; + } void set_path_info(const vectorized::PathInData& path); FieldAggregationMethod aggregation() const { return _aggregation; } vectorized::AggregateFunctionPtr get_aggregate_function_union( @@ -213,6 +218,7 @@ class TabletColumn : public MetadataAdder { std::string _aggregation_name; bool _is_nullable = false; bool _is_auto_increment = false; + bool _is_on_update_current_timestamp {false}; bool _has_default_value = false; std::string _default_value; diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index b045f7d1e8e779..81c5e9a03b8e2f 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -863,7 +863,7 @@ Status NewJsonReader::_set_column_value(rapidjson::Value& objectValue, Block& bl } return Status::OK(); } - _process_skip_bitmap_mark(slot_desc, column_ptr, block, cur_row_count, valid); + _set_skip_bitmap_mark(slot_desc, column_ptr, block, cur_row_count, valid); column_ptr->insert_default(); } else { // not found, filling with default value @@ -1712,7 +1712,7 @@ Status NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val } return Status::OK(); } - _process_skip_bitmap_mark(slot_desc, column_ptr, block, cur_row_count, valid); + _set_skip_bitmap_mark(slot_desc, column_ptr, block, cur_row_count, valid); column_ptr->insert_default(); } else { RETURN_IF_ERROR(_fill_missing_column(slot_desc, _serdes[i], column_ptr, valid)); @@ -2231,8 +2231,8 @@ void NewJsonReader::_append_empty_skip_bitmap_value(Block& block, size_t cur_row DCHECK(skip_bitmap_col_ptr->size() == cur_row_count + 1); } -void NewJsonReader::_process_skip_bitmap_mark(SlotDescriptor* slot_desc, IColumn* column_ptr, - Block& block, size_t cur_row_count, bool* valid) { +void NewJsonReader::_set_skip_bitmap_mark(SlotDescriptor* slot_desc, IColumn* column_ptr, + Block& block, size_t cur_row_count, bool* valid) { // we record the missing column's column unique id in skip bitmap // to indicate which columns need to do the alignment process auto* skip_bitmap_nullable_col_ptr = assert_cast( @@ -2241,12 +2241,7 @@ void NewJsonReader::_process_skip_bitmap_mark(SlotDescriptor* slot_desc, IColumn assert_cast(skip_bitmap_nullable_col_ptr->get_nested_column_ptr().get()); DCHECK(skip_bitmap_col_ptr->size() == cur_row_count + 1); auto& skip_bitmap = skip_bitmap_col_ptr->get_data().back(); - if (!slot_desc->is_auto_increment()) { - // For auto-increment column, it will always have a valid value when in SegmentWriter. - // Either the row specifies it, or its value is filled with generated value. So never mark the - // auto-increment column in skip bitmap - skip_bitmap.add(slot_desc->col_unique_id()); - } + skip_bitmap.add(slot_desc->col_unique_id()); } void NewJsonReader::_collect_profile_before_close() { diff --git a/be/src/vec/exec/format/json/new_json_reader.h b/be/src/vec/exec/format/json/new_json_reader.h index 81b00c0af5a164..fe74b756140993 100644 --- a/be/src/vec/exec/format/json/new_json_reader.h +++ b/be/src/vec/exec/format/json/new_json_reader.h @@ -205,8 +205,8 @@ class NewJsonReader : public GenericReader { // in `_simdjson_handle_simple_json` and `_vhandle_simple_json` (which will be used when jsonpaths is not specified) bool _should_process_skip_bitmap_col() const { return skip_bitmap_col_idx != -1; } void _append_empty_skip_bitmap_value(Block& block, size_t cur_row_count); - void _process_skip_bitmap_mark(SlotDescriptor* slot_desc, IColumn* column_ptr, Block& block, - size_t cur_row_count, bool* valid); + void _set_skip_bitmap_mark(SlotDescriptor* slot_desc, IColumn* column_ptr, Block& block, + size_t cur_row_count, bool* valid); RuntimeState* _state = nullptr; RuntimeProfile* _profile = nullptr; ScannerCounter* _counter = nullptr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 83f17ac581bee5..ead905e9ed2e8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -623,6 +623,7 @@ public TColumn toThrift() { tColumn.setIsKey(this.isKey); tColumn.setIsAllowNull(this.isAllowNull); tColumn.setIsAutoIncrement(this.isAutoInc); + tColumn.setIsOnUpdateCurrentTimestamp(this.hasOnUpdateDefaultValue); // keep compatibility tColumn.setDefaultValue(this.realDefaultValue == null ? this.defaultValue : this.realDefaultValue); tColumn.setVisible(visible); diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index 494f80224f6cda..9816ec403928ed 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -344,6 +344,8 @@ message ColumnPB { // only reference by variant sparse columns optional int32 parent_unique_id = 23; optional int32 be_exec_version = 24; + // this field is only used during flexible partial update load + optional bool is_on_update_current_timestamp = 25 [default = false]; } // Dictionary of Schema info, to reduce TabletSchemaCloudPB fdb kv size diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index ab10cd6abeca0c..6fb1fc5c39c769 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -43,6 +43,7 @@ struct TColumn { 18: optional bool is_auto_increment = false; 19: optional i32 cluster_key_id = -1 20: optional i32 be_exec_version = -1 + 21: optional bool is_on_update_current_timestamp = false } struct TSlotDescriptor { diff --git a/regression-test/data/fault_injection_p0/flexible/test3.json b/regression-test/data/fault_injection_p0/flexible/test3.json deleted file mode 100644 index 9d6f6258cf7216..00000000000000 --- a/regression-test/data/fault_injection_p0/flexible/test3.json +++ /dev/null @@ -1,5 +0,0 @@ -{"k":1,"v1":10,"v2":999} -{"k":6,"v2":654,"v4":321} -{"k":2,"v1":10,"v2":888} -{"k":7,"v1":777,"v2":777} -{"k":3,"v1":10,"v2":777} \ No newline at end of file diff --git a/regression-test/data/fault_injection_p0/flexible/test4.json b/regression-test/data/fault_injection_p0/flexible/test4.json deleted file mode 100644 index 13124151b231fb..00000000000000 --- a/regression-test/data/fault_injection_p0/flexible/test4.json +++ /dev/null @@ -1,5 +0,0 @@ -{"k":1,"v1":5,"v4":-1} -{"k":5,"v2":11111,"v4":111111} -{"k":6,"v3":111111} -{"k":2,"v1":20,"v4":-1} -{"k":3,"v4":-1} \ No newline at end of file diff --git a/regression-test/data/fault_injection_p0/flexible/test_fleixble_partial_update_publish_conflict_seq.out b/regression-test/data/fault_injection_p0/flexible/test_fleixble_partial_update_publish_conflict_seq.out deleted file mode 100644 index 28ba5c4a8662f2..00000000000000 --- a/regression-test/data/fault_injection_p0/flexible/test_fleixble_partial_update_publish_conflict_seq.out +++ /dev/null @@ -1,38 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -1 1 1 1 1 -2 2 2 2 2 -4 4 4 4 4 -5 5 5 5 5 -6 6 6 6 6 - --- !sql -- -1 10 999 1 1 3,4,5 -2 20 888 2 -1 2,3,5 -3 10 777 \N -1 1,2,3,5,8 -4 4 4 4 4 -5 5 11111 5 111111 1,3,5,8 -6 6 654 111111 321 1,2,4,5,8 -7 777 777 \N \N 3,4,5 - --- !inspect -- -1 1 1 1 1 1 2 -1 10 999 1 1 10 3 3,4,5 -1 5 1 1 -1 5 4 2,3,5 -1 10 999 1 1 10 4 3,4,5 -2 2 2 2 2 2 2 -2 10 888 2 2 10 3 3,4,5 -2 20 2 2 -1 20 4 2,3,5 -2 20 888 2 -1 20 4 2,3,5 -3 10 777 \N \N 10 3 3,4,5 -3 \N \N \N -1 \N 4 1,2,3,5,8 -3 10 777 \N -1 10 4 1,2,3,5,8 -4 4 4 4 4 4 2 -5 5 5 5 5 5 2 -5 5 11111 5 111111 5 4 1,3,5,8 -6 6 6 6 6 6 2 -6 6 654 6 321 6 3 1,3,5,8 -6 6 6 111111 6 6 4 1,2,4,5,8 -6 6 654 111111 321 6 4 1,2,4,5,8 -7 777 777 \N \N 777 3 3,4,5 - diff --git a/regression-test/data/unique_with_mow_p0/flexible/debug.json b/regression-test/data/unique_with_mow_p0/flexible/debug.json new file mode 100644 index 00000000000000..43641658524cc7 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/debug.json @@ -0,0 +1 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/debug2.json b/regression-test/data/unique_with_mow_p0/flexible/debug2.json new file mode 100644 index 00000000000000..d9b55cce992831 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/debug2.json @@ -0,0 +1 @@ +{"k":1,"v1":999,"v2":1234,"v3":567} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/debug3.json b/regression-test/data/unique_with_mow_p0/flexible/debug3.json new file mode 100644 index 00000000000000..f88daa8e0befad --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/debug3.json @@ -0,0 +1 @@ +{"k":1,"v4":777,"v5":8888} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/debug4.json b/regression-test/data/unique_with_mow_p0/flexible/debug4.json new file mode 100644 index 00000000000000..68243719ae0e4f --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/debug4.json @@ -0,0 +1 @@ +{"k":1,"v1":999,"v2":-1,"v4":-2} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/default1.json b/regression-test/data/unique_with_mow_p0/flexible/default1.json new file mode 100644 index 00000000000000..6ceccd625229b3 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/default1.json @@ -0,0 +1,8 @@ +{"k":0,"v1":999} +{"k":1,"v2":999} +{"k":2,"v3":"xyz"} +{"k":3,"v4":999} +{"k":10,"v1":999} +{"k":11,"v2":999} +{"k":12,"v3":"abcd"} +{"k":13,"v4":999} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/default2.json b/regression-test/data/unique_with_mow_p0/flexible/default2.json new file mode 100644 index 00000000000000..07d61c35012c4e --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/default2.json @@ -0,0 +1,4 @@ +{"k":10,"v1":999,"t1":"2020-01-01 00:00:00"} +{"k":11,"v2":888,"t2":"2020-01-01 00:00:00.123456"} +{"k":12,"v2":888,"t3":"2020-01-01 00:00:00"} +{"k":13,"v1":777,"v2":666} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/default3.json b/regression-test/data/unique_with_mow_p0/flexible/default3.json new file mode 100644 index 00000000000000..8ef7a5767407d9 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/default3.json @@ -0,0 +1,5 @@ +{"k":0,"v1":999} +{"k":1,"v2":888} +{"k":2,"v1":777,"t1":"2021-12-12 00:00:00"} +{"k":3,"v2":666,"t2":"2021-12-12 00:00:00"} +{"k":4,"v1":555,"t1":"2021-12-12 00:00:00","t2":"2021-12-12 00:00:00"} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete10.json b/regression-test/data/unique_with_mow_p0/flexible/delete10.json new file mode 100644 index 00000000000000..671665f5a0674a --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete10.json @@ -0,0 +1,39 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1,"v2":1111} +{"k":1,"__DORIS_DELETE_SIGN__":1,"v1":15} +{"k":1,"v3":1111,"v1":17} +{"k":2,"__DORIS_DELETE_SIGN__":1,"v1":15} +{"k":2,"v3":2222,"v1":30} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"v2":2222} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3,"v2":3333} +{"k":3,"__DORIS_DELETE_SIGN__":1,"v1":15} +{"k":3,"v3":3333,"v1":25} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":4,"v2":4444} +{"k":4,"__DORIS_DELETE_SIGN__":1,"v1":40} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5,"v2":5555} +{"k":5,"v3":5555,"v1":60} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v2":6666} +{"k":6,"__DORIS_DELETE_SIGN__":1,"v1":40} +{"k":7,"v3":7777,"v1":75} +{"k":7,"__DORIS_DELETE_SIGN__":1} +{"k":7,"v2":7777} +{"k":8,"__DORIS_DELETE_SIGN__":1} +{"k":8,"__DORIS_DELETE_SIGN__":1,"v1":90} +{"k":8,"v3":8888,"v1":100} +{"k":9,"v2":9999} +{"k":9,"__DORIS_DELETE_SIGN__":1,"v1":80} +{"k":9,"v3":9999,"v1":100} +{"k":10,"v2":10101010} +{"k":10,"__DORIS_DELETE_SIGN__":1,"v1":110} +{"k":10,"v3":9999999,"v1":90} +{"k":10,"v3":10101010,"v1":120} +{"k":10,"__DORIS_DELETE_SIGN__":1,"v1":110} +{"k":20,"__DORIS_DELETE_SIGN__":1} +{"k":20,"v2":20202020} +{"k":20,"__DORIS_DELETE_SIGN__":1,"v1":300} +{"k":20,"v3":20202020,"v1":400} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete11.json b/regression-test/data/unique_with_mow_p0/flexible/delete11.json new file mode 100644 index 00000000000000..d647524e8d30a0 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete11.json @@ -0,0 +1,12 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1,"v2":1111} +{"k":1,"__DORIS_DELETE_SIGN__":1,"v1":40} +{"k":1,"v3":1111,"v1":50} +{"k":2,"__DORIS_DELETE_SIGN__":1,"v1":15} +{"k":2,"v3":2222,"v1":40} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"v2":2222} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3,"v2":3333} +{"k":3,"__DORIS_DELETE_SIGN__":1,"v1":15} +{"k":3,"v3":3333,"v1":25} diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete3.json b/regression-test/data/unique_with_mow_p0/flexible/delete3.json new file mode 100644 index 00000000000000..15ea4ee5f498f0 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete3.json @@ -0,0 +1,27 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1,"v1":11111} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"v1":22222} +{"k":2,"v2":33333,"v5":44444} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"v3":22222,"__DORIS_DELETE_SIGN__":1} +{"k":3,"v1":3333,"v3":44444} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":5,"v1":5555,"v2":5555} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5,"v1":6666,"v3":77777} +{"k":6,"v1":5555,"v2":5555} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v1":6666,"v3":77777} +{"k":6,"v3":888} +{"k":6,"v3":66666,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v4":9999} +{"k":6,"v5":9999} +{"k":6,"v2":9999} +{"k":6,"v3":66666,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v3":66666,"__DORIS_DELETE_SIGN__":1} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete4.json b/regression-test/data/unique_with_mow_p0/flexible/delete4.json new file mode 100644 index 00000000000000..1adffb9de4eff5 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete4.json @@ -0,0 +1,27 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":1,"v1":11111} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":3,"v1":3333,"v3":44444} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v1":5555,"v2":5555} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"v1":22222} +{"k":2,"v2":33333,"v5":44444} +{"k":5,"v1":5555,"v2":5555} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"v3":22222,"__DORIS_DELETE_SIGN__":1} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v3":888} +{"k":6,"v3":66666,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v4":9999} +{"k":6,"v5":9999} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v1":6666,"v3":77777} +{"k":5,"v1":6666,"v3":77777} +{"k":6,"v2":9999} +{"k":6,"v3":66666,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v3":66666,"__DORIS_DELETE_SIGN__":1} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete5.json b/regression-test/data/unique_with_mow_p0/flexible/delete5.json new file mode 100644 index 00000000000000..9ef1004af8164d --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete5.json @@ -0,0 +1,62 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1, "v2":111} +{"k":1, "v3":111} +{"k":2, "v3":222} +{"k":2, "v4":222,"v5":222} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3, "v2":333} +{"k":3, "v3":333} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":4, "v3":444} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":4, "v3":444} +{"k":4, "v4":444,"v2":444} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5, "v2":555} +{"k":5, "v3":555} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5, "v4":555} +{"k":5, "v3":555} +{"k":6, "v3":666} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6, "v3":666} +{"k":6, "v4":666,"v2":666} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":7,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":5} +{"k":7,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":8} +{"k":7, "v2":111,"__DORIS_SEQUENCE_COL__":12} +{"k":7, "v3":111,"__DORIS_SEQUENCE_COL__":15} +{"k":8, "v3":222,"__DORIS_SEQUENCE_COL__":12} +{"k":8, "v4":222,"v5":222,"__DORIS_SEQUENCE_COL__":15} +{"k":8,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":16} +{"k":8,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":17} +{"k":9,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":4} +{"k":9,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":5} +{"k":9, "v2":333,"__DORIS_SEQUENCE_COL__":10} +{"k":9, "v3":333,"__DORIS_SEQUENCE_COL__":12} +{"k":9,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":15} +{"k":10, "v3":444,"__DORIS_SEQUENCE_COL__":1} +{"k":10,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":5} +{"k":10,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":10} +{"k":10, "v3":444,"__DORIS_SEQUENCE_COL__":15} +{"k":10, "v4":444,"v2":444,"__DORIS_SEQUENCE_COL__":25} +{"k":11,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":1} +{"k":11,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":7} +{"k":11, "v2":555,"__DORIS_SEQUENCE_COL__":15} +{"k":11, "v3":555,"__DORIS_SEQUENCE_COL__":18} +{"k":11,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":20} +{"k":11, "v4":555,"__DORIS_SEQUENCE_COL__":30} +{"k":11, "v3":555,"__DORIS_SEQUENCE_COL__":43} +{"k":12, "v3":666,"__DORIS_SEQUENCE_COL__":3} +{"k":12,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":6} +{"k":12,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":10} +{"k":12, "v3":666,"__DORIS_SEQUENCE_COL__":13} +{"k":12, "v4":666,"v2":666,"__DORIS_SEQUENCE_COL__":18} +{"k":12,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":20} diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete6.json b/regression-test/data/unique_with_mow_p0/flexible/delete6.json new file mode 100644 index 00000000000000..091c6ae1ecc0de --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete6.json @@ -0,0 +1,42 @@ +{"k":7,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":5} +{"k":7,"v3":7777,"__DORIS_SEQUENCE_COL__":4} +{"k":7,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":8} +{"k":7, "v2":111,"__DORIS_SEQUENCE_COL__":12} +{"k":7,"v4":7777,"__DORIS_SEQUENCE_COL__":10} +{"k":7, "v3":111,"__DORIS_SEQUENCE_COL__":15} +{"k":7,"v4":7777,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":10} +{"k":8, "v3":222,"__DORIS_SEQUENCE_COL__":12} +{"k":8, "v3":8888,"__DORIS_SEQUENCE_COL__":1} +{"k":8, "v4":222,"v5":222,"__DORIS_SEQUENCE_COL__":15} +{"k":8,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":16} +{"k":8,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":17} +{"k":8, "v3":8888,"__DORIS_SEQUENCE_COL__":12} +{"k":9,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":4} +{"k":9, "v3":9999,"__DORIS_SEQUENCE_COL__":3} +{"k":9,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":5} +{"k":9, "v2":333,"__DORIS_SEQUENCE_COL__":10} +{"k":9, "v3":9999,"__DORIS_SEQUENCE_COL__":9,"__DORIS_DELETE_SIGN__":1} +{"k":9, "v3":333,"__DORIS_SEQUENCE_COL__":12} +{"k":9,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":15} +{"k":9, "v4":9999,"__DORIS_SEQUENCE_COL__":10} +{"k":10, "v3":444,"__DORIS_SEQUENCE_COL__":1} +{"k":10,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":5} +{"k":10,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":10} +{"k":10, "v3":444,"__DORIS_SEQUENCE_COL__":15} +{"k":10, "v3":1010101,"__DORIS_SEQUENCE_COL__":10,"__DORIS_DELETE_SIGN__":1} +{"k":10, "v4":444,"v2":444,"__DORIS_SEQUENCE_COL__":25} +{"k":11,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":1} +{"k":11,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":7} +{"k":11, "v2":555,"__DORIS_SEQUENCE_COL__":15} +{"k":11, "v3":555,"__DORIS_SEQUENCE_COL__":18} +{"k":11,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":20} +{"k":11, "v4":555,"__DORIS_SEQUENCE_COL__":30} +{"k":11, "v4":111111111,"__DORIS_SEQUENCE_COL__":29,"__DORIS_DELETE_SIGN__":1} +{"k":11, "v3":555,"__DORIS_SEQUENCE_COL__":43} +{"k":12, "v3":666,"__DORIS_SEQUENCE_COL__":3} +{"k":12,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":6} +{"k":12,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":10} +{"k":12, "v3":666,"__DORIS_SEQUENCE_COL__":13} +{"k":12, "v4":666,"v2":666,"__DORIS_SEQUENCE_COL__":18} +{"k":12,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":20} +{"k":12, "v4":666,"v2":666,"__DORIS_SEQUENCE_COL__":19} diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete7.json b/regression-test/data/unique_with_mow_p0/flexible/delete7.json new file mode 100644 index 00000000000000..6e209fbdafec51 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete7.json @@ -0,0 +1,39 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1,"v2":1111} +{"k":1,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":15} +{"k":1,"v3":1111,"__DORIS_SEQUENCE_COL__":17} +{"k":2,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":15} +{"k":2,"v3":2222,"__DORIS_SEQUENCE_COL__":30} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"v2":2222} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3,"v2":3333} +{"k":3,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":15} +{"k":3,"v3":3333,"__DORIS_SEQUENCE_COL__":25} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":4,"v2":4444} +{"k":4,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":40} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5,"v2":5555} +{"k":5,"v3":5555,"__DORIS_SEQUENCE_COL__":60} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6,"v2":6666} +{"k":6,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":40} +{"k":7,"v3":7777,"__DORIS_SEQUENCE_COL__":75} +{"k":7,"__DORIS_DELETE_SIGN__":1} +{"k":7,"v2":7777} +{"k":8,"__DORIS_DELETE_SIGN__":1} +{"k":8,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":90} +{"k":8,"v3":8888,"__DORIS_SEQUENCE_COL__":100} +{"k":9,"v2":9999} +{"k":9,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":80} +{"k":9,"v3":9999,"__DORIS_SEQUENCE_COL__":100} +{"k":10,"v2":10101010} +{"k":10,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":110} +{"k":10,"v3":9999999,"__DORIS_SEQUENCE_COL__":90} +{"k":10,"v3":10101010,"__DORIS_SEQUENCE_COL__":120} +{"k":10,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":110} +{"k":20,"__DORIS_DELETE_SIGN__":1} +{"k":20,"v2":20202020} +{"k":20,"__DORIS_DELETE_SIGN__":1,"__DORIS_SEQUENCE_COL__":300} +{"k":20,"v3":20202020,"__DORIS_SEQUENCE_COL__":400} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete8.json b/regression-test/data/unique_with_mow_p0/flexible/delete8.json new file mode 100644 index 00000000000000..5ce7787c016fee --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete8.json @@ -0,0 +1,62 @@ +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1,"__DORIS_DELETE_SIGN__":1} +{"k":1, "v2":111} +{"k":1, "v3":111} +{"k":2, "v3":222} +{"k":2, "v4":222,"v5":222} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":2,"__DORIS_DELETE_SIGN__":1} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":3, "v2":333} +{"k":3, "v3":333} +{"k":3,"__DORIS_DELETE_SIGN__":1} +{"k":4, "v3":444} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":4,"__DORIS_DELETE_SIGN__":1} +{"k":4, "v3":444} +{"k":4, "v4":444,"v2":444} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5, "v2":555} +{"k":5, "v3":555} +{"k":5,"__DORIS_DELETE_SIGN__":1} +{"k":5, "v4":555} +{"k":5, "v3":555} +{"k":6, "v3":666} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":6, "v3":666} +{"k":6, "v4":666,"v2":666} +{"k":6,"__DORIS_DELETE_SIGN__":1} +{"k":7,"__DORIS_DELETE_SIGN__":1,"v1":5} +{"k":7,"__DORIS_DELETE_SIGN__":1,"v1":8} +{"k":7, "v2":111,"v1":12} +{"k":7, "v3":111,"v1":15} +{"k":8, "v3":222,"v1":12} +{"k":8, "v4":222,"v5":222,"v1":15} +{"k":8,"__DORIS_DELETE_SIGN__":1,"v1":16} +{"k":8,"__DORIS_DELETE_SIGN__":1,"v1":17} +{"k":9,"__DORIS_DELETE_SIGN__":1,"v1":4} +{"k":9,"__DORIS_DELETE_SIGN__":1,"v1":5} +{"k":9, "v2":333,"v1":10} +{"k":9, "v3":333,"v1":12} +{"k":9,"__DORIS_DELETE_SIGN__":1,"v1":15} +{"k":10, "v3":444,"v1":1} +{"k":10,"__DORIS_DELETE_SIGN__":1,"v1":5} +{"k":10,"__DORIS_DELETE_SIGN__":1,"v1":10} +{"k":10, "v3":444,"v1":15} +{"k":10, "v4":444,"v2":444,"v1":25} +{"k":11,"__DORIS_DELETE_SIGN__":1,"v1":1} +{"k":11,"__DORIS_DELETE_SIGN__":1,"v1":7} +{"k":11, "v2":555,"v1":15} +{"k":11, "v3":555,"v1":18} +{"k":11,"__DORIS_DELETE_SIGN__":1,"v1":20} +{"k":11, "v4":555,"v1":30} +{"k":11, "v3":555,"v1":43} +{"k":12, "v3":666,"v1":3} +{"k":12,"__DORIS_DELETE_SIGN__":1,"v1":6} +{"k":12,"__DORIS_DELETE_SIGN__":1,"v1":10} +{"k":12, "v3":666,"v1":13} +{"k":12, "v4":666,"v2":666,"v1":18} +{"k":12,"__DORIS_DELETE_SIGN__":1,"v1":20} diff --git a/regression-test/data/unique_with_mow_p0/flexible/delete9.json b/regression-test/data/unique_with_mow_p0/flexible/delete9.json new file mode 100644 index 00000000000000..175f15cc51c7dd --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/delete9.json @@ -0,0 +1,42 @@ +{"k":7,"__DORIS_DELETE_SIGN__":1,"v1":5} +{"k":7,"v3":7777,"v1":4} +{"k":7,"__DORIS_DELETE_SIGN__":1,"v1":8} +{"k":7, "v2":111,"v1":12} +{"k":7,"v4":7777,"v1":10} +{"k":7, "v3":111,"v1":15} +{"k":7,"v4":7777,"__DORIS_DELETE_SIGN__":1,"v1":10} +{"k":8, "v3":222,"v1":12} +{"k":8, "v3":8888,"v1":1} +{"k":8, "v4":222,"v5":222,"v1":15} +{"k":8,"__DORIS_DELETE_SIGN__":1,"v1":16} +{"k":8,"__DORIS_DELETE_SIGN__":1,"v1":17} +{"k":8, "v3":8888,"v1":12} +{"k":9,"__DORIS_DELETE_SIGN__":1,"v1":4} +{"k":9, "v3":9999,"v1":3} +{"k":9,"__DORIS_DELETE_SIGN__":1,"v1":5} +{"k":9, "v2":333,"v1":10} +{"k":9, "v3":9999,"v1":9,"__DORIS_DELETE_SIGN__":1} +{"k":9, "v3":333,"v1":12} +{"k":9,"__DORIS_DELETE_SIGN__":1,"v1":15} +{"k":9, "v4":9999,"v1":10} +{"k":10, "v3":444,"v1":1} +{"k":10,"__DORIS_DELETE_SIGN__":1,"v1":5} +{"k":10,"__DORIS_DELETE_SIGN__":1,"v1":10} +{"k":10, "v3":444,"v1":15} +{"k":10, "v3":1010101,"v1":10,"__DORIS_DELETE_SIGN__":1} +{"k":10, "v4":444,"v2":444,"v1":25} +{"k":11,"__DORIS_DELETE_SIGN__":1,"v1":1} +{"k":11,"__DORIS_DELETE_SIGN__":1,"v1":7} +{"k":11, "v2":555,"v1":15} +{"k":11, "v3":555,"v1":18} +{"k":11,"__DORIS_DELETE_SIGN__":1,"v1":20} +{"k":11, "v4":555,"v1":30} +{"k":11, "v4":111111111,"v1":29,"__DORIS_DELETE_SIGN__":1} +{"k":11, "v3":555,"v1":43} +{"k":12, "v3":666,"v1":3} +{"k":12,"__DORIS_DELETE_SIGN__":1,"v1":6} +{"k":12,"__DORIS_DELETE_SIGN__":1,"v1":10} +{"k":12, "v3":666,"v1":13} +{"k":12, "v4":666,"v2":666,"v1":18} +{"k":12,"__DORIS_DELETE_SIGN__":1,"v1":20} +{"k":12, "v4":666,"v2":666,"v1":19} diff --git a/regression-test/data/fault_injection_p0/flexible/test1.json b/regression-test/data/unique_with_mow_p0/flexible/publish/test1.json similarity index 100% rename from regression-test/data/fault_injection_p0/flexible/test1.json rename to regression-test/data/unique_with_mow_p0/flexible/publish/test1.json diff --git a/regression-test/data/fault_injection_p0/flexible/test2.json b/regression-test/data/unique_with_mow_p0/flexible/publish/test2.json similarity index 100% rename from regression-test/data/fault_injection_p0/flexible/test2.json rename to regression-test/data/unique_with_mow_p0/flexible/publish/test2.json diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test3.json b/regression-test/data/unique_with_mow_p0/flexible/publish/test3.json new file mode 100644 index 00000000000000..6947b241fb56d3 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test3.json @@ -0,0 +1,11 @@ +{"k":1,"v1":10,"v2":999} +{"k":2,"v1":10,"v2":999} +{"k":3,"v1":10,"v2":999} +{"k":4,"v2":999} +{"k":14,"v1":10,"v2":888} +{"k":15,"v1":10,"v2":888} +{"k":16,"v1":10,"v4":888} +{"k":17,"v4":888} +{"k":27,"v1":10,"v4":777,"__DORIS_DELETE_SIGN__":1} +{"k":28,"v1":10,"v4":777,"__DORIS_DELETE_SIGN__":1} +{"k":29,"v1":10,"v4":777,"__DORIS_DELETE_SIGN__":1} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test4.json b/regression-test/data/unique_with_mow_p0/flexible/publish/test4.json new file mode 100644 index 00000000000000..54e47fb46c0dc7 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test4.json @@ -0,0 +1,12 @@ +{"k":1,"v1":5,"v4":-1} +{"k":2,"v1":20,"v4":-1} +{"k":3,"v4":-1} +{"k":4,"v1":10,"v4":-1} +{"k":14,"v1":5,"v4":-2,"__DORIS_DELETE_SIGN__":1} +{"k":15,"v1":20,"v4":-2,"__DORIS_DELETE_SIGN__":1} +{"k":16,"v4":-2,"__DORIS_DELETE_SIGN__":1} +{"k":17,"v1":10,"v4":-2,"__DORIS_DELETE_SIGN__":1} +{"k":27,"v1":5,"v4":-3} +{"k":28,"v1":20,"v4":-3} +{"k":29,"v4":-3} +{"k":30,"v1":10,"v4":-3} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test5.json b/regression-test/data/unique_with_mow_p0/flexible/publish/test5.json new file mode 100644 index 00000000000000..6022dda8b2393a --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test5.json @@ -0,0 +1,2 @@ +{"k":1, "v3":888, "v4":777} +{"k":2, "v3":444, "v5":555} diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test6.json b/regression-test/data/unique_with_mow_p0/flexible/publish/test6.json new file mode 100644 index 00000000000000..5d4138caa1cbce --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test6.json @@ -0,0 +1,2 @@ +{"k":1, "v1":10,"v3":888, "v4":777} +{"k":2, "v2":444, "v4":555} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.out b/regression-test/data/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.out new file mode 100644 index 00000000000000..0bdcb740ed6a83 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 100 200 20 +2 2 2 22 + +-- !sql -- +1 100 \N 20 2 +1 \N 200 21 3 +1 100 200 20 3 +2 2 2 22 4 + diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test_f_auto_inc_compaction.out b/regression-test/data/unique_with_mow_p0/flexible/publish/test_f_auto_inc_compaction.out new file mode 100644 index 00000000000000..dbf2cbab4c171f --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test_f_auto_inc_compaction.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +5 5 5 5 +6 6 6 6 + +-- !sql -- +2 2 2 +4 4 4 +5 5 5 + +-- !check_auto_inc_dup -- + +-- !after_compaction -- +2 2 2 +4 4 4 +5 5 5 +99 99 99 + +-- !check_auto_inc_dup -- + +-- !sql -- +2 2 2 +3 100 \N +4 4 4 +5 5 5 +6 \N 600 +99 99 99 + +-- !check_auto_inc_dup -- + +-- !check_auto_inc_val -- +0 + diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.out b/regression-test/data/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.out new file mode 100644 index 00000000000000..6cede5aa3eb58b --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 100 1 1 1 1 +2 100 2 2 2 2 +3 100 3 3 3 3 +4 100 4 4 4 4 + +-- !sql -- +1 100 987 77777 1234 \N +2 200 987 77777 1234 \N +3 100 987 77777 3 3 +4 100 987 77777 1234 \N + +-- !inspect -- +1 100 1 1 1 1 100 2 0 +1 100 1 1 1 1 100 3 2,3,4,5 1 +1 100 987 77777 1 1 100 4 1,4,5,6,9 0 +1 100 987 77777 1234 \N 100 4 1,4,5,6,9 0 +2 100 2 2 2 2 100 2 0 +2 200 2 2 2 2 200 3 2,3,4,5 1 +2 100 987 77777 2 2 100 4 1,4,5,6,9 0 +2 200 987 77777 1234 \N 200 4 1,4,5,6,9 0 +3 100 3 3 3 3 100 2 0 +3 50 \N 9876 1234 \N 50 3 2,3,4,5 1 +3 100 987 77777 3 3 100 4 1,4,5,6,9 0 +4 100 4 4 4 4 100 2 0 +4 100 4 4 4 4 100 3 1,2,3,4,5,9 1 +4 100 987 77777 4 4 100 4 1,4,5,6,9 0 +4 100 987 77777 1234 \N 100 4 1,4,5,6,9 0 + diff --git a/regression-test/data/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.out b/regression-test/data/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.out new file mode 100644 index 00000000000000..efac872b6b7bbc --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.out @@ -0,0 +1,103 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 1 1 +2 1 2 2 2 +3 1 3 3 3 +4 1 4 4 4 +14 1 14 14 14 +15 1 15 15 15 +16 1 16 16 16 +17 1 17 17 17 +27 1 27 27 27 +28 1 28 28 28 +29 1 29 29 29 + +-- !sql -- +1 10 999 1 1 3,4,5 10 4 +2 20 999 2 -1 2,3,5 20 4 +3 10 999 3 -1 1,2,3,5,8 10 4 +4 10 999 4 -1 2,3,5 10 4 +14 10 888 14 14 3,4,5 10 4 +28 20 \N \N -3 2,3,5 20 4 +29 10 \N \N -3 1,2,3,5,8 10 4 +30 10 \N \N -3 2,3,5 10 4 + +-- !skip_delete_sign -- +1 10 999 1 1 3,4,5 10 0 4 +2 20 999 2 -1 2,3,5 20 0 4 +3 10 999 3 -1 1,2,3,5,8 10 0 4 +4 10 999 4 -1 2,3,5 10 0 4 +14 10 888 14 14 3,4,5 10 0 4 +15 20 888 15 -2 2,3 20 1 4 +16 10 16 16 -2 1,2,3,8 10 1 4 +17 10 17 17 -2 2,3 10 1 4 +27 10 27 27 777 2,3 10 1 4 +28 20 \N \N -3 2,3,5 20 0 4 +29 10 \N \N -3 1,2,3,5,8 10 0 4 +30 10 \N \N -3 2,3,5 10 0 4 + +-- !inspect -- +1 1 1 1 1 1 0 2 0 +1 10 999 1 1 10 0 3 3,4,5 0 +1 10 999 1 1 10 0 4 3,4,5 0 +1 5 1 1 -1 5 0 4 2,3,5 0 +2 1 2 2 2 1 0 2 0 +2 10 999 2 2 10 0 3 3,4,5 0 +2 20 2 2 -1 20 0 4 2,3,5 0 +2 20 999 2 -1 20 0 4 2,3,5 0 +3 1 3 3 3 1 0 2 0 +3 10 999 3 3 10 0 3 3,4,5 0 +3 10 999 3 -1 10 0 4 1,2,3,5,8 0 +3 1 3 3 -1 1 0 4 1,2,3,5,8 0 +4 1 4 4 4 1 0 2 0 +4 1 999 4 4 1 0 3 1,3,4,5,8 0 +4 10 4 4 -1 10 0 4 2,3,5 0 +4 10 999 4 -1 10 0 4 2,3,5 0 +14 1 14 14 14 1 0 2 0 +14 10 888 14 14 10 0 3 3,4,5 0 +14 5 14 14 -2 5 1 4 2,3 1 +14 10 888 14 14 10 0 4 3,4,5 0 +15 1 15 15 15 1 0 2 0 +15 10 888 15 15 10 0 3 3,4,5 0 +15 20 15 15 -2 20 1 4 2,3 1 +15 20 888 15 -2 20 1 4 2,3 1 +16 1 16 16 16 1 0 2 0 +16 10 16 16 888 10 0 3 2,3,5 0 +16 10 16 16 -2 10 1 4 1,2,3,8 1 +16 1 16 16 -2 1 1 4 1,2,3,8 1 +17 1 17 17 17 1 0 2 0 +17 1 17 17 888 1 0 3 1,2,3,5,8 0 +17 10 17 17 -2 10 1 4 2,3 1 +17 10 17 17 -2 10 1 4 2,3 1 +27 1 27 27 27 1 0 2 0 +27 10 27 27 777 10 1 3 2,3 1 +27 10 27 27 777 10 1 4 2,3 1 +27 5 27 27 -3 5 0 4 2,3,5 0 +28 1 28 28 28 1 0 2 0 +28 10 28 28 777 10 1 3 2,3 1 +28 20 28 28 -3 20 0 4 2,3,5 0 +28 20 \N \N -3 20 0 4 2,3,5 0 +29 1 29 29 29 1 0 2 0 +29 10 29 29 777 10 1 3 2,3 1 +29 10 \N \N -3 10 0 4 1,2,3,5,8 0 +29 1 29 29 -3 1 0 4 1,2,3,5,8 0 +30 10 \N \N -3 10 0 4 2,3,5 0 + +-- !sql1 -- +1 99 99 99 1 +2 20 2 2 2 + +-- !sql2 -- +1 99 99 99 1 +2 20 444 2 555 + +-- !inspect -- +1 10 1 1 1 10 2 0 +1 99 99 99 \N 99 3 0 +1 99 99 99 1 99 3 0 +1 10 \N 888 777 10 4 2,5 0 +1 99 99 99 1 99 4 0 +2 20 2 2 2 20 2 0 +2 \N 444 \N 555 \N 4 1,3,5,8 0 +2 20 444 2 555 20 4 1,3,5,8 0 + diff --git a/regression-test/data/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.out b/regression-test/data/unique_with_mow_p0/flexible/publish/test_flexible_partial_update_publish_conflict.out similarity index 82% rename from regression-test/data/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.out rename to regression-test/data/unique_with_mow_p0/flexible/publish/test_flexible_partial_update_publish_conflict.out index ff143148ef9781..d9298ee945c56f 100644 --- a/regression-test/data/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.out +++ b/regression-test/data/unique_with_mow_p0/flexible/publish/test_flexible_partial_update_publish_conflict.out @@ -17,3 +17,10 @@ 6 1999 2777 3777 1234 \N 1,4,6 7 \N \N 3777 1234 5777 1,4,6 +-- !sql1 -- +1 99 99 99 1 1 +2 2 2 2 2 2 +-- !sql2 -- +1 99 99 888 777 1 +2 2 2 444 2 555 + diff --git a/regression-test/data/unique_with_mow_p0/flexible/test4.json b/regression-test/data/unique_with_mow_p0/flexible/test4.json index 774c728995df4d..7ed84ac2834d64 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test4.json +++ b/regression-test/data/unique_with_mow_p0/flexible/test4.json @@ -13,5 +13,5 @@ {"k": 6, "v2": 666, "__DORIS_SEQUENCE_COL__": 30} {"k": 6, "v5": 666} {"k": 6, "v1": 123} -{"k": 6, "v3": 666, "__DORIS_SEQUENCE_COL__": 20} +{"k": 6, "v3": 8970, "__DORIS_SEQUENCE_COL__": 70} {"k": 6, "v2": 123} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/test5.json b/regression-test/data/unique_with_mow_p0/flexible/test5.json index a3a4c48d8fe73f..cec7e1b7e3bb43 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test5.json +++ b/regression-test/data/unique_with_mow_p0/flexible/test5.json @@ -13,7 +13,7 @@ {"k": 7, "v5": 666} {"k": 3, "v2": 123} {"k": 4, "v3": 123, "v4": 987} -{"k": 7, "v1": 123} +{"k": 7, "v1": 123, "v3": 7890} {"k": 3, "v4": 123} {"k": 4, "v4": 555, "v2": 777, "__DORIS_SEQUENCE_COL__": 39} {"k": 7, "v3": 666, "__DORIS_SEQUENCE_COL__": 20} diff --git a/regression-test/data/unique_with_mow_p0/flexible/test6.json b/regression-test/data/unique_with_mow_p0/flexible/test6.json index f86b1ed255294f..582c798d92dfa2 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test6.json +++ b/regression-test/data/unique_with_mow_p0/flexible/test6.json @@ -8,10 +8,31 @@ {"k": 2, "v5": 777} {"k": 2, "v4": 666, "v6": 15} {"k": 2, "v4": 777} +{"k": 5, "v1": 999, "v6": 10} +{"k": 5, "v2": 888, "v6": 30} +{"k": 5, "v4": 5678, "v6": 40} +{"k": 5, "v1": 1234, "v6": 90} +{"k": 5, "v2": 1234, "v3": 7890} +{"k": 5, "v5": 1234, "v6": 100} +{"k": 5, "v1": 333, "v6": 80} +{"k": 5, "v1": 111222} {"k": 6, "v1": 666, "v6": 10} {"k": 6, "v4": 666} {"k": 6, "v2": 666, "v6": 30} {"k": 6, "v5": 666} {"k": 6, "v1": 123} -{"k": 6, "v3": 666, "v6": 20} -{"k": 6, "v2": 123} \ No newline at end of file +{"k": 6, "v3": 7890, "v6": 70} +{"k": 6, "v2": 123} +{"k": 8, "v1": 123, "v2":456} +{"k": 8, "v1": 999, "v3": 999, "v6": 50} +{"k": 8, "v2": 8888, "v6": 70} +{"k": 8, "v3": 23456, "v5": 999} +{"k": 9, "v1": 999, "v6": 10} +{"k": 9, "v2": 888, "v6": 30} +{"k": 9, "v3": 777, "v6": 60} +{"k": 9, "v1": 1234, "v6": 90} +{"k": 9, "v2": 1234, "v3": 7890} +{"k": 9, "v4": 5678} +{"k": 9, "v5": 1234, "v6": 100} +{"k": 9, "v1": 333, "v8": 80} +{"k": 9, "v1": 111222} \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/flexible/test7.json b/regression-test/data/unique_with_mow_p0/flexible/test7.json index 710e97e691e1e3..6499e40b3e4e65 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test7.json +++ b/regression-test/data/unique_with_mow_p0/flexible/test7.json @@ -13,7 +13,7 @@ {"k": 7, "v5": 666} {"k": 3, "v2": 123} {"k": 4, "v3": 123, "v4": 987} -{"k": 7, "v1": 123} +{"k": 7, "v1": 123, "v3": 7890} {"k": 3, "v4": 123} {"k": 4, "v4": 555, "v2": 777, "v6": 39} {"k": 7, "v3": 666, "v6": 30} diff --git a/regression-test/data/unique_with_mow_p0/flexible/test_f_seq_read_from_old.out b/regression-test/data/unique_with_mow_p0/flexible/test_f_seq_read_from_old.out new file mode 100644 index 00000000000000..3e35864037f36a --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/test_f_seq_read_from_old.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1000 1 1 1 1 + +-- !sql1 -- + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 1,2,3,4,5,9 + +-- !sql2 -- +1 1000 \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 1,2,3,4,5,9 +1 1000 \N 9876 777 8888 1000 0 4 1,2,3,6,9 + +-- !sql3 -- +1 1000 \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 1,2,3,4,5,9 +1 1000 \N 9876 777 8888 1000 0 4 1,2,3,6,9 +1 999 -1 9876 -2 \N 999 0 5 3,5,6 + +-- !sql -- +1 1000 1 1 1 1 + +-- !sql1 -- + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 1,2,3,4,5,10 + +-- !sql2 -- +1 1000 \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 1,2,3,4,5,10 +1 1000 \N 9876 777 8888 1000 0 4 1,2,3,6,10 + +-- !sql3 -- +1 1000 \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 1,2,3,4,5,10 +1 1000 \N 9876 777 8888 1000 0 4 1,2,3,6,10 +1 999 -1 9876 -2 \N 999 0 5 3,5,6 + diff --git a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out index ccdf59b7f7b1d8..b5e2c1d647a10e 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out +++ b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.out @@ -11,11 +11,11 @@ 11 -- !autoinc_key_3 -- -\N 111 111 1234 \N 1,4,5,6 -\N 9876 666 1234 \N 1,2,4,6 -\N 9876 888 222 \N 1,2,5,6 -43 9876 99 20 \N 2,5,6 -999 9876 777 1234 \N 2,4,5,6 +\N 111 111 1234 \N 0,1,4,5,6 +\N 9876 666 1234 \N 0,1,2,4,6 +\N 9876 888 222 \N 0,1,2,5,6 +43 9876 99 20 \N 0,2,5,6 +999 9876 777 1234 \N 0,2,4,5,6 -- !autoinc_key_4 -- 1000 10 0 0 0 0 2,3,4,5,6 @@ -35,7 +35,7 @@ -- !autoinc_val_2 -- 0 0 0 0 0 0 -1 10 1 1 1 1 2,4,5,6 +1 10 1 1 1 1 2,3,4,5,6 2 2 20 20 2 25 1,4,6 3 3 3 30 3 3 1,2,4,5,6 4 4 4 4 4 4 @@ -43,8 +43,8 @@ 9 \N 99 77 1234 \N 1,4,5,6 -- !autoinc_val_3 -- -8 \N 88 1234 \N 1,4,5,6 -10 \N 9876 666 555 1,2,6 +8 \N 88 1234 \N 1,3,4,5,6 +10 \N 9876 666 555 1,2,3,6 -- !autoinc_val_4 -- 2 @@ -61,11 +61,11 @@ 11 -- !autoinc_key_3 -- -\N 111 111 1234 \N 1,4,5,6 -\N 9876 666 1234 \N 1,2,4,6 -\N 9876 888 222 \N 1,2,5,6 -43 9876 99 20 \N 2,5,6 -999 9876 777 1234 \N 2,4,5,6 +\N 111 111 1234 \N 0,1,4,5,6 +\N 9876 666 1234 \N 0,1,2,4,6 +\N 9876 888 222 \N 0,1,2,5,6 +43 9876 99 20 \N 0,2,5,6 +999 9876 777 1234 \N 0,2,4,5,6 -- !autoinc_key_4 -- 1000 10 0 0 0 0 2,3,4,5,6 @@ -85,7 +85,7 @@ -- !autoinc_val_2 -- 0 0 0 0 0 0 -1 10 1 1 1 1 2,4,5,6 +1 10 1 1 1 1 2,3,4,5,6 2 2 20 20 2 25 1,4,6 3 3 3 30 3 3 1,2,4,5,6 4 4 4 4 4 4 @@ -93,8 +93,8 @@ 9 \N 99 77 1234 \N 1,4,5,6 -- !autoinc_val_3 -- -8 \N 88 1234 \N 1,4,5,6 -10 \N 9876 666 555 1,2,6 +8 \N 88 1234 \N 1,3,4,5,6 +10 \N 9876 666 555 1,2,3,6 -- !autoinc_val_4 -- 2 diff --git a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_default_value.out b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_default_value.out new file mode 100644 index 00000000000000..1c7ce7ab1dfe9b --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_default_value.out @@ -0,0 +1,137 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +0 0 0 0 0 +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 + +-- !sql -- +0 999 0 0 0 +1 1 999 1 1 +2 2 2 xyz 2 +3 3 3 3 999 +10 999 9876 test \N +11 \N 999 test \N +12 \N 9876 abcd \N +13 \N 9876 test 999 + +-- !sql -- +11 \N 888 +12 \N 888 +13 777 666 + +-- !sql -- +10 999 \N +12 \N 888 +13 777 666 + +-- !sql -- +10 999 \N +11 \N 888 +13 777 666 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +0 0 0 2020-01-01T00:00 2020-01-01T00:00 +1 1 1 2020-01-01T00:00 2020-01-01T00:00 +2 2 2 2020-01-01T00:00 2020-01-01T00:00 +3 3 3 2020-01-01T00:00 2020-01-01T00:00 +4 4 4 2020-01-01T00:00 2020-01-01T00:00 + +-- !sql -- +0 999 0 +1 1 888 +2 777 2 +3 3 666 +4 555 4 + +-- !sql -- +0 999 0 +1 1 888 +3 3 666 + +-- !sql -- +0 999 0 +1 1 888 +2 777 2 + +-- !sql -- +0 999 0 +1 1 888 + +-- !sql -- +0 0 0 0 0 +1 1 1 1 1 +2 2 2 2 2 +3 3 3 3 3 + +-- !sql -- +0 999 0 0 0 +1 1 999 1 1 +2 2 2 xyz 2 +3 3 3 3 999 +10 999 9876 test \N +11 \N 999 test \N +12 \N 9876 abcd \N +13 \N 9876 test 999 + +-- !sql -- +11 \N 888 +12 \N 888 +13 777 666 + +-- !sql -- +10 999 \N +12 \N 888 +13 777 666 + +-- !sql -- +10 999 \N +11 \N 888 +13 777 666 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +2 + +-- !sql -- +0 0 0 2020-01-01T00:00 2020-01-01T00:00 +1 1 1 2020-01-01T00:00 2020-01-01T00:00 +2 2 2 2020-01-01T00:00 2020-01-01T00:00 +3 3 3 2020-01-01T00:00 2020-01-01T00:00 +4 4 4 2020-01-01T00:00 2020-01-01T00:00 + +-- !sql -- +0 999 0 +1 1 888 +2 777 2 +3 3 666 +4 555 4 + +-- !sql -- +0 999 0 +1 1 888 +3 3 666 + +-- !sql -- +0 999 0 +1 1 888 +2 777 2 + +-- !sql -- +0 999 0 +1 1 888 + diff --git a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.out b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.out index 47e6a61de8a929..3b2cf56dcd5f95 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.out +++ b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.out @@ -1,169 +1,505 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !no_seq_col_1 -- -0 0 0 0 0 0 -1 1 1 1 1 1 -2 2 2 2 2 2 -3 3 3 3 3 3 -4 4 4 4 4 4 -5 5 5 5 5 5 +0 0 0 0 0 0 +1 1 1 1 1 1 +2 2 2 2 2 2 +3 3 3 3 3 3 +4 4 4 4 4 4 +5 5 5 5 5 5 -- !no_seq_col_2 -- -0 0 0 0 0 0 -2 2 20 2 2 25 1,3,4,6 -4 43 4 99 20 4 2,5,6 -5 5 5 5 5 \N 1,2,3,4,6 -6 999 9876 777 1234 \N 2,4,5,6 +0 0 0 0 0 0 +2 2 20 2 2 25 +4 43 4 99 20 4 +5 5 5 5 5 \N +6 999 9876 777 1234 \N -- !inspect -- -0 0 0 0 0 0 0 -1 \N 9876 0 1234 \N 1 1,2,3,4,5 -1 1 1 1 1 1 0 -2 2 2 2 2 2 0 -2 2 20 2 2 25 0 1,3,4,6 -3 \N 9876 0 1234 \N 1 1,2,3,4,5 -3 3 3 3 3 3 0 -4 4 4 4 4 4 0 -4 43 4 99 20 4 0 2,5,6 -5 5 5 5 5 \N 0 1,2,3,4,6 -5 5 5 5 5 5 0 -6 999 9876 777 1234 \N 0 2,4,5,6 -7 \N 9876 0 1234 \N 1 1,2,3,4,5 +0 0 0 0 0 0 0 2 +1 1 1 1 1 1 0 2 +1 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +2 2 2 2 2 2 0 2 +2 2 20 2 2 25 0 1,3,4,6 3 +3 3 3 3 3 3 0 2 +3 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +4 4 4 4 4 4 0 2 +4 43 4 99 20 4 0 2,5,6 3 +5 5 5 5 5 5 0 2 +5 5 5 5 5 \N 0 1,2,3,4,6 3 +6 999 9876 777 1234 \N 0 2,4,5,6 3 +7 \N 9876 0 1234 \N 1 1,2,3,4,5 3 -- !no_seq_col_3 -- -0 0 0 0 0 0 -2 2 20 2 2 25 1,3,4,6 -4 43 4 99 20 4 2,5,6 -5 5 5 5 5 \N 1,2,3,4,6 -6 999 9876 777 1234 \N 2,4,5,6 +0 0 0 0 0 0 +2 2 20 2 2 25 +4 43 4 99 20 4 +5 5 5 5 5 \N +6 999 9876 777 1234 \N -- !inspect -- -0 0 0 0 0 0 0 -1 \N 9876 0 1234 \N 1 1,2,3,4,5 -1 \N 9876 0 1234 \N 1 1,2,3,4,5 -1 1 1 1 1 1 0 -2 2 2 2 2 2 0 -2 2 20 2 2 25 0 1,3,4,6 -3 \N 9876 0 1234 \N 1 1,2,3,4,5 -3 3 3 3 3 3 0 -4 4 4 4 4 4 0 -4 43 4 99 20 4 0 2,5,6 -5 5 5 5 5 \N 0 1,2,3,4,6 -5 5 5 5 5 5 0 -6 999 9876 777 1234 \N 0 2,4,5,6 -7 \N 9876 0 1234 \N 1 1,2,3,4,5 -7 \N 9876 0 1234 \N 1 1,2,3,4,5 +0 0 0 0 0 0 0 2 +1 1 1 1 1 1 0 2 +1 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +1 \N 9876 0 1234 \N 1 1,2,3,4,5 4 +2 2 2 2 2 2 0 2 +2 2 20 2 2 25 0 1,3,4,6 3 +3 3 3 3 3 3 0 2 +3 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +4 4 4 4 4 4 0 2 +4 43 4 99 20 4 0 2,5,6 3 +5 5 5 5 5 5 0 2 +5 5 5 5 5 \N 0 1,2,3,4,6 3 +6 999 9876 777 1234 \N 0 2,4,5,6 3 +7 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +7 \N 9876 0 1234 \N 1 1,2,3,4,5 4 -- !seq_map_col_1 -- -0 0 0 0 0 0 0 -1 1 1 1 1 1 1 -2 2 2 2 2 2 2 -3 3 3 3 3 3 3 -4 4 4 4 4 4 4 -5 5 5 5 5 5 5 +0 0 0 0 0 0 0 +1 1 1 1 1 1 1 +2 2 2 2 2 2 2 +3 3 3 3 3 3 3 +4 4 4 4 4 4 4 +5 5 5 5 5 5 5 -- !seq_map_col_2 -- -0 0 0 0 0 0 0 0 -2 2 20 2 2 25 0 25 1,3,4,6 -4 43 4 99 20 4 0 4 2,5,6,9 -5 5 5 5 5 5 0 5 -6 999 9876 777 1234 \N 0 \N 2,4,5,6,9 +0 0 0 0 0 0 0 0 +2 2 20 2 2 25 0 25 +4 43 4 99 20 4 0 4 +5 5 5 5 5 5 0 5 +6 999 9876 777 1234 \N 0 \N -- !inspect -- -0 0 0 0 0 0 0 0 -1 1 1 1 1 1 0 1 -1 1 1 1 1 1 1 1 1,2,3,4,5,9 -2 2 2 2 2 2 0 2 -2 2 20 2 2 25 0 25 1,3,4,6 -3 3 3 3 3 3 0 3 -3 3 3 3 3 3 1 3 1,2,3,4,5,9 -4 4 4 4 4 4 0 4 -4 43 4 99 20 4 0 4 2,5,6,9 -5 5 5 5 5 \N 0 \N 1,2,3,4,6 -5 5 5 5 5 5 0 5 -6 999 9876 777 1234 \N 0 \N 2,4,5,6,9 -7 \N 9876 0 1234 \N 1 \N 1,2,3,4,5,9 +0 0 0 0 0 0 0 0 2 +1 1 1 1 1 1 0 1 2 +1 1 1 1 1 1 1 1 1,2,3,4,5,9 3 +2 2 2 2 2 2 0 2 2 +2 2 20 2 2 25 0 25 1,3,4,6 3 +3 3 3 3 3 3 0 3 2 +3 3 3 3 3 3 1 3 1,2,3,4,5,9 3 +4 4 4 4 4 4 0 4 2 +4 43 4 99 20 4 0 4 2,5,6,9 3 +5 5 5 5 5 5 0 5 2 +5 \N 9876 0 1234 \N 0 \N 1,2,3,4,6 3 +6 999 9876 777 1234 \N 0 \N 2,4,5,6,9 3 +7 \N 9876 0 1234 \N 1 \N 1,2,3,4,5,9 3 --- !no_seq_col_1 -- -0 0 0 0 0 0 -1 1 1 1 1 1 -2 2 2 2 2 2 -3 3 3 3 3 3 -4 4 4 4 4 4 -5 5 5 5 5 5 +-- !insert_after_delete_3_1 -- +0 0 0 0 0 0 +1 1 1 1 1 1 +2 2 2 2 2 2 +3 3 3 3 3 3 +4 4 4 4 4 4 +5 5 5 5 5 5 +6 6 6 6 6 6 +7 7 7 7 7 7 +8 8 8 8 8 8 +9 9 9 9 9 9 --- !no_seq_col_2 -- -0 0 0 0 0 0 -2 2 20 2 2 25 1,3,4,6 -4 43 4 99 20 4 2,5,6 -5 5 5 5 5 \N 1,2,3,4,6 -6 999 9876 777 1234 \N 2,4,5,6 +-- !insert_after_delete_3_1 -- +0 0 0 0 0 0 0 +1 11111 9876 0 1234 \N 0 +5 6666 9876 77777 1234 \N 0 +7 7 7 7 7 7 0 +8 8 8 8 8 8 0 +9 9 9 9 9 9 0 -- !inspect -- -0 0 0 0 0 0 0 -1 \N 9876 0 1234 \N 1 1,2,3,4,5 -1 1 1 1 1 1 0 -2 2 2 2 2 2 0 -2 2 20 2 2 25 0 1,3,4,6 -3 \N 9876 0 1234 \N 1 1,2,3,4,5 -3 3 3 3 3 3 0 -4 4 4 4 4 4 0 -4 43 4 99 20 4 0 2,5,6 -5 5 5 5 5 \N 0 1,2,3,4,6 -5 5 5 5 5 5 0 -6 999 9876 777 1234 \N 0 2,4,5,6 -7 \N 9876 0 1234 \N 1 1,2,3,4,5 +0 0 0 0 0 0 0 2 +1 1 1 1 1 1 0 2 +1 11111 9876 0 1234 \N 0 2,3,4,5,6 3 +2 2 2 2 2 2 0 2 +2 \N 9876 22222 1234 \N 1 1,2,4,5 3 +3 3 3 3 3 3 0 2 +3 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +4 4 4 4 4 4 0 2 +4 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +5 5 5 5 5 5 0 2 +5 6666 9876 77777 1234 \N 0 2,4,5,6 3 +6 6 6 6 6 6 0 2 +6 \N 9876 66666 1234 \N 1 1,2,4,5 3 +7 7 7 7 7 7 0 2 +8 8 8 8 8 8 0 2 +9 9 9 9 9 9 0 2 --- !no_seq_col_3 -- -0 0 0 0 0 0 -2 2 20 2 2 25 1,3,4,6 -4 43 4 99 20 4 2,5,6 -5 5 5 5 5 \N 1,2,3,4,6 -6 999 9876 777 1234 \N 2,4,5,6 +-- !insert_after_delete_3_1 -- +0 0 0 0 0 0 +1 1 1 1 1 1 +2 2 2 2 2 2 +3 3 3 3 3 3 +4 4 4 4 4 4 +5 5 5 5 5 5 +6 6 6 6 6 6 +7 7 7 7 7 7 +8 8 8 8 8 8 +9 9 9 9 9 9 + +-- !insert_after_delete_3_1 -- +0 0 0 0 0 0 0 +1 11111 9876 0 1234 \N 0 +5 6666 9876 77777 1234 \N 0 +7 7 7 7 7 7 0 +8 8 8 8 8 8 0 +9 9 9 9 9 9 0 -- !inspect -- -0 0 0 0 0 0 0 -1 \N 9876 0 1234 \N 1 1,2,3,4,5 -1 \N 9876 0 1234 \N 1 1,2,3,4,5 -1 1 1 1 1 1 0 -2 2 2 2 2 2 0 -2 2 20 2 2 25 0 1,3,4,6 -3 \N 9876 0 1234 \N 1 1,2,3,4,5 -3 3 3 3 3 3 0 -4 4 4 4 4 4 0 -4 43 4 99 20 4 0 2,5,6 -5 5 5 5 5 \N 0 1,2,3,4,6 -5 5 5 5 5 5 0 -6 999 9876 777 1234 \N 0 2,4,5,6 -7 \N 9876 0 1234 \N 1 1,2,3,4,5 -7 \N 9876 0 1234 \N 1 1,2,3,4,5 +0 0 0 0 0 0 0 2 +1 1 1 1 1 1 0 2 +1 11111 9876 0 1234 \N 0 2,3,4,5,6 3 +2 2 2 2 2 2 0 2 +2 \N 9876 22222 1234 \N 1 1,2,4,5 3 +3 3 3 3 3 3 0 2 +3 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +4 4 4 4 4 4 0 2 +4 \N 9876 0 1234 \N 1 1,2,3,4,5 3 +5 5 5 5 5 5 0 2 +5 6666 9876 77777 1234 \N 0 2,4,5,6 3 +6 6 6 6 6 6 0 2 +6 \N 9876 66666 1234 \N 1 1,2,4,5 3 +7 7 7 7 7 7 0 2 +8 8 8 8 8 8 0 2 +9 9 9 9 9 9 0 2 --- !seq_map_col_1 -- -0 0 0 0 0 0 0 -1 1 1 1 1 1 1 -2 2 2 2 2 2 2 -3 3 3 3 3 3 3 -4 4 4 4 4 4 4 -5 5 5 5 5 5 5 +-- !insert_after_delete_3_2_1 -- +0 0 0 0 0 0 \N +1 1 1 1 1 1 \N +2 2 2 2 2 2 \N +3 3 3 3 3 3 \N +4 4 4 4 4 4 \N +5 5 5 5 5 5 \N +6 6 6 6 6 6 \N +7 7 7 7 7 7 \N +8 8 8 8 8 8 \N +9 9 9 9 9 9 \N +10 10 10 10 10 10 \N +11 11 11 11 11 11 \N +12 12 12 12 12 12 \N +13 13 13 13 13 13 \N +14 14 14 14 14 14 \N --- !seq_map_col_2 -- -0 0 0 0 0 0 0 0 -2 2 20 2 2 25 0 25 1,3,4,6 -4 43 4 99 20 4 0 4 2,5,6,10 -5 5 5 5 5 5 0 5 -6 999 9876 777 1234 \N 0 \N 2,4,5,6,10 +-- !insert_after_delete_3_2_2 -- +0 0 0 0 0 0 0 +1 \N 111 111 1234 9753 0 +4 \N 444 444 444 9753 0 +5 \N 9876 555 555 9753 0 +7 \N 111 111 1234 9753 0 +10 \N 444 444 444 9753 0 +11 \N 9876 555 555 9753 0 +13 13 13 13 13 13 0 +14 14 14 14 14 14 0 + +-- !inspect -- +0 0 0 0 0 0 \N 0 2 +1 1 1 1 1 1 \N 0 2 +1 \N 111 111 1234 9753 \N 0 1,4,5,6 3 +2 2 2 2 2 2 \N 0 2 +2 2 2 2 2 2 \N 1 1,2,3,4,5,9 3 +3 3 3 3 3 3 \N 0 2 +3 3 3 3 3 3 \N 1 1,2,3,4,5,9 3 +4 4 4 4 4 4 \N 0 2 +4 \N 444 444 444 9753 \N 0 1,5,6 3 +5 5 5 5 5 5 \N 0 2 +5 \N 9876 555 555 9753 \N 0 1,2,5,6 3 +6 6 6 6 6 6 \N 0 2 +6 6 6 6 6 6 \N 1 1,2,3,4,5,9 3 +7 7 7 7 7 7 \N 0 2 +7 \N 111 111 1234 9753 15 0 1,4,5,6 3 +8 8 8 8 8 8 \N 0 2 +8 8 8 8 8 8 17 1 1,2,3,4,5 3 +9 9 9 9 9 9 \N 0 2 +9 9 9 9 9 9 15 1 1,2,3,4,5 3 +10 10 10 10 10 10 \N 0 2 +10 \N 444 444 444 9753 25 0 1,5,6 3 +11 11 11 11 11 11 \N 0 2 +11 \N 9876 555 555 9753 43 0 1,2,5,6 3 +12 12 12 12 12 12 \N 0 2 +12 12 12 12 12 12 20 1 1,2,3,4,5 3 +13 13 13 13 13 13 \N 0 2 +14 14 14 14 14 14 \N 0 2 + +-- !insert_after_delete_3_2_3 -- +0 0 0 0 0 0 \N +1 1 1 1 1 1 \N +2 2 2 2 2 2 \N +3 3 3 3 3 3 \N +4 4 4 4 4 4 \N +5 5 5 5 5 5 \N +6 6 6 6 6 6 \N +7 7 7 7 7 7 \N +8 8 8 8 8 8 \N +9 9 9 9 9 9 \N +10 10 10 10 10 10 \N +11 11 11 11 11 11 \N +12 12 12 12 12 12 \N +13 13 13 13 13 13 \N +14 14 14 14 14 14 \N + +-- !insert_after_delete_3_2_3 -- +0 0 0 0 0 0 0 +1 1 1 1 1 1 0 +2 2 2 2 2 2 0 +3 3 3 3 3 3 0 +4 4 4 4 4 4 0 +5 5 5 5 5 5 0 +6 6 6 6 6 6 0 +7 \N 111 111 1234 9753 0 +10 \N 444 444 444 9753 0 +11 \N 9876 555 555 9753 0 +13 13 13 13 13 13 0 +14 14 14 14 14 14 0 + +-- !inspect -- +0 0 0 0 0 0 \N 0 2 +1 1 1 1 1 1 \N 0 2 +2 2 2 2 2 2 \N 0 2 +3 3 3 3 3 3 \N 0 2 +4 4 4 4 4 4 \N 0 2 +5 5 5 5 5 5 \N 0 2 +6 6 6 6 6 6 \N 0 2 +7 7 7 7 7 7 \N 0 2 +7 \N 111 111 1234 9753 15 0 1,4,5,6 3 +8 8 8 8 8 8 \N 0 2 +8 8 8 8 8 8 17 1 1,2,3,4,5 3 +9 9 9 9 9 9 \N 0 2 +9 9 9 9 9 9 15 1 1,2,3,4,5 3 +10 10 10 10 10 10 \N 0 2 +10 \N 444 444 444 9753 25 0 1,5,6 3 +11 11 11 11 11 11 \N 0 2 +11 \N 9876 555 555 9753 43 0 1,2,5,6 3 +12 12 12 12 12 12 \N 0 2 +12 12 12 12 12 12 20 1 1,2,3,4,5 3 +13 13 13 13 13 13 \N 0 2 +14 14 14 14 14 14 \N 0 2 + +-- !insert_after_delete_3_2_4 -- +0 0 0 0 0 0 0 +1 1 1 1 1 1 10 +2 2 2 2 2 2 20 +3 3 3 3 3 3 30 +4 4 4 4 4 4 40 +5 5 5 5 5 5 50 +6 6 6 6 6 6 60 +7 7 7 7 7 7 70 +8 8 8 8 8 8 80 +9 9 9 9 9 9 90 +10 10 10 10 10 10 100 +11 11 11 11 11 11 110 +12 12 12 12 12 12 120 + +-- !insert_after_delete_3_2_4 -- +0 0 0 0 0 0 0 0 +1 \N 9876 1111 1234 9753 17 0 +2 \N 2222 5432 1234 9753 20 0 +3 \N 3333 5432 1234 9753 30 0 +5 \N 5555 5555 1234 9753 60 0 +6 \N 6666 5432 1234 9753 60 0 +7 \N 7777 5432 1234 9753 70 0 +8 \N 9876 8888 1234 9753 100 0 +9 9 9999 9999 9 9 100 0 +10 \N 9876 10101010 1234 9753 120 0 +11 11 11 11 11 11 110 0 +12 12 12 12 12 12 120 0 +20 \N 9876 20202020 1234 9753 400 0 + +-- !inspect -- +0 0 0 0 0 0 0 0 2 +1 1 1 1 1 1 10 0 2 +1 \N 9876 1111 1234 9753 17 0 1,2,4,5,6 3 +2 2 2 2 2 2 20 0 2 +2 \N 2222 5432 1234 9753 20 0 1,3,4,5,6 3 +3 3 3 3 3 3 30 0 2 +3 \N 3333 5432 1234 9753 30 0 1,3,4,5,6 3 +4 4 4 4 4 4 40 0 2 +4 4 4 4 4 4 40 1 1,2,3,4,5 3 +5 5 5 5 5 5 50 0 2 +5 \N 5555 5555 1234 9753 60 0 1,4,5,6 3 +6 6 6 6 6 6 60 0 2 +6 \N 6666 5432 1234 9753 60 0 1,3,4,5,6 3 +7 7 7 7 7 7 70 0 2 +7 \N 7777 5432 1234 9753 70 0 1,3,4,5,6 3 +8 8 8 8 8 8 80 0 2 +8 \N 9876 8888 1234 9753 100 0 1,2,4,5,6 3 +9 9 9 9 9 9 90 0 2 +9 9 9999 9999 9 9 100 0 1,4,5,6 3 +10 10 10 10 10 10 100 0 2 +10 \N 9876 10101010 1234 9753 120 0 1,2,4,5,6 3 +11 11 11 11 11 11 110 0 2 +12 12 12 12 12 12 120 0 2 +20 \N 9876 20202020 1234 9753 400 0 1,2,4,5,6 3 + +-- !insert_after_delete_3_3_1 -- +0 \N 0 0 0 0 \N +1 \N 1 1 1 1 \N +2 \N 2 2 2 2 \N +3 \N 3 3 3 3 \N +4 \N 4 4 4 4 \N +5 \N 5 5 5 5 \N +6 \N 6 6 6 6 \N +7 \N 7 7 7 7 \N +8 \N 8 8 8 8 \N +9 \N 9 9 9 9 \N +10 \N 10 10 10 10 \N +11 \N 11 11 11 11 \N +12 \N 12 12 12 12 \N +13 \N 13 13 13 13 \N +14 \N 14 14 14 14 \N + +-- !insert_after_delete_3_3_2 -- +0 \N 0 0 0 0 0 +1 \N 111 111 1234 9753 0 +4 \N 444 444 444 9753 0 +5 \N 9876 555 555 9753 0 +7 15 111 111 1234 9753 0 +10 25 444 444 444 9753 0 +11 43 9876 555 555 9753 0 +13 \N 13 13 13 13 0 +14 \N 14 14 14 14 0 + +-- !inspect -- +0 \N 0 0 0 0 \N 0 2 +1 \N 1 1 1 1 \N 0 2 +1 \N 111 111 1234 9753 \N 0 1,4,5,6 3 +2 \N 2 2 2 2 \N 0 2 +2 \N 2 2 2 2 \N 1 1,2,3,4,5,9 3 +3 \N 3 3 3 3 \N 0 2 +3 \N 3 3 3 3 \N 1 1,2,3,4,5,9 3 +4 \N 4 4 4 4 \N 0 2 +4 \N 444 444 444 9753 \N 0 1,5,6 3 +5 \N 5 5 5 5 \N 0 2 +5 \N 9876 555 555 9753 \N 0 1,2,5,6 3 +6 \N 6 6 6 6 \N 0 2 +6 \N 6 6 6 6 \N 1 1,2,3,4,5,9 3 +7 \N 7 7 7 7 \N 0 2 +7 15 111 111 1234 9753 15 0 4,5,6 3 +8 \N 8 8 8 8 \N 0 2 +8 17 8 8 8 8 17 1 2,3,4,5 3 +9 \N 9 9 9 9 \N 0 2 +9 15 9 9 9 9 15 1 2,3,4,5 3 +10 \N 10 10 10 10 \N 0 2 +10 25 444 444 444 9753 25 0 5,6 3 +11 \N 11 11 11 11 \N 0 2 +11 43 9876 555 555 9753 43 0 2,5,6 3 +12 \N 12 12 12 12 \N 0 2 +12 20 12 12 12 12 20 1 2,3,4,5 3 +13 \N 13 13 13 13 \N 0 2 +14 \N 14 14 14 14 \N 0 2 + +-- !insert_after_delete_3_3_3 -- +0 \N 0 0 0 0 \N +1 \N 1 1 1 1 \N +2 \N 2 2 2 2 \N +3 \N 3 3 3 3 \N +4 \N 4 4 4 4 \N +5 \N 5 5 5 5 \N +6 \N 6 6 6 6 \N +7 \N 7 7 7 7 \N +8 \N 8 8 8 8 \N +9 \N 9 9 9 9 \N +10 \N 10 10 10 10 \N +11 \N 11 11 11 11 \N +12 \N 12 12 12 12 \N +13 \N 13 13 13 13 \N +14 \N 14 14 14 14 \N + +-- !insert_after_delete_3_3_4 -- +0 \N 0 0 0 0 0 +1 \N 1 1 1 1 0 +2 \N 2 2 2 2 0 +3 \N 3 3 3 3 0 +4 \N 4 4 4 4 0 +5 \N 5 5 5 5 0 +6 \N 6 6 6 6 0 +7 15 111 111 1234 9753 0 +10 25 444 444 444 9753 0 +11 43 9876 555 555 9753 0 +13 \N 13 13 13 13 0 +14 \N 14 14 14 14 0 + +-- !inspect -- +0 \N 0 0 0 0 \N 0 2 +1 \N 1 1 1 1 \N 0 2 +2 \N 2 2 2 2 \N 0 2 +3 \N 3 3 3 3 \N 0 2 +4 \N 4 4 4 4 \N 0 2 +5 \N 5 5 5 5 \N 0 2 +6 \N 6 6 6 6 \N 0 2 +7 \N 7 7 7 7 \N 0 2 +7 15 111 111 1234 9753 15 0 4,5,6 3 +8 \N 8 8 8 8 \N 0 2 +8 17 8 8 8 8 17 1 2,3,4,5 3 +9 \N 9 9 9 9 \N 0 2 +9 15 9 9 9 9 15 1 2,3,4,5 3 +10 \N 10 10 10 10 \N 0 2 +10 25 444 444 444 9753 25 0 5,6 3 +11 \N 11 11 11 11 \N 0 2 +11 43 9876 555 555 9753 43 0 2,5,6 3 +12 \N 12 12 12 12 \N 0 2 +12 20 12 12 12 12 20 1 2,3,4,5 3 +13 \N 13 13 13 13 \N 0 2 +14 \N 14 14 14 14 \N 0 2 + +-- !insert_after_delete_3_3_5 -- +0 0 0 0 0 0 0 +1 10 1 1 1 1 10 +2 20 2 2 2 2 20 +3 30 3 3 3 3 30 +4 40 4 4 4 4 40 +5 50 5 5 5 5 50 +6 60 6 6 6 6 60 +7 70 7 7 7 7 70 +8 80 8 8 8 8 80 +9 90 9 9 9 9 90 +10 100 10 10 10 10 100 +11 110 11 11 11 11 110 +12 120 12 12 12 12 120 + +-- !insert_after_delete_3_3_6 -- +0 0 0 0 0 0 0 0 +1 17 9876 1111 1234 9753 17 0 +2 \N 2222 5432 1234 9753 20 0 +3 \N 3333 5432 1234 9753 30 0 +5 60 5555 5555 1234 9753 60 0 +6 \N 6666 5432 1234 9753 60 0 +7 \N 7777 5432 1234 9753 70 0 +8 100 9876 8888 1234 9753 100 0 +9 100 9999 9999 9 9 100 0 +10 120 9876 10101010 1234 9753 120 0 +11 110 11 11 11 11 110 0 +12 120 12 12 12 12 120 0 +20 400 9876 20202020 1234 9753 400 0 + +-- !inspect -- +0 0 0 0 0 0 0 0 2 +1 10 1 1 1 1 10 0 2 +1 17 9876 1111 1234 9753 17 0 2,4,5,6 3 +2 20 2 2 2 2 20 0 2 +2 \N 2222 5432 1234 9753 20 0 1,3,4,5,6 3 +3 30 3 3 3 3 30 0 2 +3 \N 3333 5432 1234 9753 30 0 1,3,4,5,6 3 +4 40 4 4 4 4 40 0 2 +4 40 4 4 4 4 40 1 2,3,4,5 3 +5 50 5 5 5 5 50 0 2 +5 60 5555 5555 1234 9753 60 0 4,5,6 3 +6 60 6 6 6 6 60 0 2 +6 \N 6666 5432 1234 9753 60 0 1,3,4,5,6 3 +7 70 7 7 7 7 70 0 2 +7 \N 7777 5432 1234 9753 70 0 1,3,4,5,6 3 +8 80 8 8 8 8 80 0 2 +8 100 9876 8888 1234 9753 100 0 2,4,5,6 3 +9 90 9 9 9 9 90 0 2 +9 100 9999 9999 9 9 100 0 4,5,6 3 +10 100 10 10 10 10 100 0 2 +10 120 9876 10101010 1234 9753 120 0 2,4,5,6 3 +11 110 11 11 11 11 110 0 2 +12 120 12 12 12 12 120 0 2 +20 400 9876 20202020 1234 9753 400 0 2,4,5,6 3 + +-- !insert_after_delete_3_4 -- +1 50 9876 1111 1234 9753 50 0 +2 30 2222 5432 1234 9753 30 0 +3 30 3333 5432 1234 9753 30 0 -- !inspect -- -0 0 0 0 0 0 0 0 -1 1 1 1 1 1 0 1 -1 1 1 1 1 1 1 1 1,2,3,4,5,10 -2 2 2 2 2 2 0 2 -2 2 20 2 2 25 0 25 1,3,4,6 -3 3 3 3 3 3 0 3 -3 3 3 3 3 3 1 3 1,2,3,4,5,10 -4 4 4 4 4 4 0 4 -4 43 4 99 20 4 0 4 2,5,6,10 -5 5 5 5 5 \N 0 \N 1,2,3,4,6 -5 5 5 5 5 5 0 5 -6 999 9876 777 1234 \N 0 \N 2,4,5,6,10 -7 \N 9876 0 1234 \N 1 \N 1,2,3,4,5,10 +1 50 9876 1111 1234 9753 50 0 2,4,5,6 2 +2 30 2222 5432 1234 9753 30 0 1,3,4,5,6,9 2 +3 30 3333 5432 1234 9753 30 0 1,3,4,5,6,9 2 diff --git a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.out b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.out index 2f86060c9eb5a8..35d6a3a6e919bc 100644 --- a/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.out +++ b/regression-test/data/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.out @@ -68,17 +68,17 @@ 3 3 3 3 3 3 30 4 4 4 4 4 4 40 5 5 5 5 5 5 50 -6 666 666 0 1234 \N 30 3,4,5,6 +6 123 123 8970 666 666 70 6 -- !seq_type_col_multi_rows_3 -- 0 0 0 0 0 0 0 1 888 888 1 1 1 20 3,4,5,6 2 2 2 777 777 777 20 1,2,6,9 -3 777 777 555 555 555 53 6 +3 777 123 123 123 3 53 5,6 4 123 123 123 987 123 40 6,9 5 5 5 5 5 5 50 -6 666 666 0 1234 \N 30 3,4,5,6 -7 666 666 0 1234 \N 30 3,4,5,6 +6 123 123 8970 666 666 70 6 +7 123 123 7890 666 666 30 6 -- !seq_map_col_no_default_val_multi_rows_1 -- 0 0 0 0 0 0 0 0 @@ -94,18 +94,22 @@ 2 2 2 777 777 777 20 20 1,2,6,7,10 3 3 3 3 3 3 30 30 4 4 4 4 4 4 40 40 -5 5 5 5 5 5 50 50 -6 666 666 0 1234 \N 30 30 3,4,5,7 +5 111222 1234 7890 5 1234 100 100 4,7 +6 123 123 7890 666 666 70 70 7 +8 999 8888 23456 1234 999 70 70 4,7 +9 111222 1234 7890 5678 1234 100 100 7 -- !seq_map_col_no_default_val_multi_rows_3 -- 0 0 0 0 0 0 0 1 888 888 1 1 1 20 3,4,5,7 2 2 2 777 777 777 20 1,2,6,7,10 -3 777 777 555 555 555 53 7 +3 777 123 123 123 3 53 5,7 4 123 123 123 987 123 40 6,7,10 -5 5 5 5 5 5 50 -6 666 666 0 1234 \N 30 3,4,5,7 -7 666 666 0 1234 \N 70 3,4,5,7 +5 111222 1234 7890 5 1234 100 4,7 +6 123 123 7890 666 666 70 7 +7 123 123 7890 666 666 70 7 +8 999 8888 23456 1234 999 70 4,7 +9 111222 1234 7890 5678 1234 100 7 -- !seq_map_col_has_default_val_multi_rows_1 -- 0 0 0 0 0 0 0 0 @@ -121,209 +125,22 @@ 2 2 2 777 777 777 20 20 1,2,6,7,10 3 3 3 3 3 3 30 30 4 4 4 4 4 4 40 40 -5 5 5 5 5 5 50 50 -6 123 123 0 666 666 60 60 3,6,7,10 +5 111222 1234 7890 5 1234 100 100 4,7 +6 123 123 7890 666 666 70 70 7 +8 123 456 23456 1234 999 80 80 4,6,7,10 +9 111222 1234 7890 5678 1234 100 100 7 -- !seq_map_col_has_default_val_multi_rows_3 -- 0 0 0 0 0 0 0 1 888 888 1 1 1 20 3,4,5,7 2 2 2 777 777 777 20 1,2,6,7,10 -3 777 777 555 555 555 53 7 +3 777 123 123 123 3 53 5,7 4 123 123 123 987 123 40 6,7,10 -5 5 5 5 5 5 50 -6 123 123 0 666 666 60 3,6,7,10 -7 666 666 0 1234 \N 70 3,4,5,7 - --- !seq1 -- -1 256 1 1 256 -2 255 2 2 255 -3 3 3 3 3 -4 4 4 4 4 - --- !seq1 -- -1 256 9999 1 256 -2 256 2 -1 256 -3 3 9999 3 3 -4 4 9999 4 4 -5 255 \N -1 255 -6 266 \N -1 266 -7 \N 9999 \N \N - --- !seq1 -- -1 256 1 1 256 -1 256 9999 1 256 -2 255 2 2 255 -2 256 2 -1 256 -3 3 3 3 3 -3 3 9999 3 3 -4 4 4 4 4 -4 4 9999 4 4 -5 255 \N -1 255 -6 266 \N -1 266 -7 \N 9999 \N \N - --- !seq2 -- -1 256 1 1 256 -2 255 2 2 255 -3 3 3 3 3 -4 4 4 4 4 - --- !seq2 -- -1 256 9999 1 256 -2 256 2 -1 256 -3 3 9999 3 3 -4 4 9999 4 4 -5 256 9999 \N 256 -6 266 \N -1 266 -7 256 9999 \N 256 - --- !seq2 -- -1 256 1 1 256 -1 256 9999 1 256 -2 255 2 2 255 -2 256 2 -1 256 -3 3 3 3 3 -3 3 9999 3 3 -4 4 4 4 4 -4 4 9999 4 4 -5 256 9999 \N 256 -6 266 \N -1 266 -7 256 9999 \N 256 - --- !seq_map_no_default_val1 -- -0 0 0 0 0 0 0 -1 1 1 1 1 10 10 -2 2 2 2 2 20 20 -3 3 3 3 3 30 30 -4 4 4 4 4 40 40 -5 5 5 5 5 50 50 - --- !seq_map_no_default_val2 -- -0 0 0 0 0 0 0 -1 1 1 1 1 10 10 -2 2 222 2 2 25 25 1,3,4,6 -3 3 3 333 3 30 30 1,2,4,5,6,10 -4 411 \N 433 444 50 50 6 -5 5 5 5 5 50 50 -6 611 9876 633 1234 \N \N 2,4,5,6,10 -7 \N 9876 733 1234 300 300 1,2,4,6 - --- !seq_map_has_default_val1 -- -0 0 0 0 0 0 0 -1 1 1 1 1 10 10 -2 2 2 2 2 20 20 -3 3 3 3 3 30 30 -4 4 4 4 4 40 40 -5 5 5 5 5 50 50 - --- !seq_map_has_default_val2 -- -0 0 0 0 0 0 0 -1 1 1 1 1 10 10 -2 2 222 2 2 25 25 1,3,4,6 -3 3 3 333 3 30 30 1,2,4,5,6,10 -4 411 \N 433 444 50 50 6 -5 5 5 5 5 50 50 -6 611 9876 633 1234 31 31 2,4,5,6,10 -7 \N 9876 733 1234 300 300 1,2,4,6 - --- !seq_type_col1 -- -0 0 0 0 0 0 0 -1 1 1 1 1 1 10 -2 2 2 2 2 2 20 -3 3 3 3 3 3 30 -4 4 4 4 4 4 40 -5 5 5 5 5 5 50 - --- !seq_type_col2 -- -0 0 0 0 0 0 0 -1 1 1 1 1 1 10 -2 2 222 2 2 25 25 1,3,4,6 -3 3 3 333 3 3 30 1,2,4,5,6,10 -4 411 \N 433 444 50 50 6 -5 5 5 5 5 5 50 -6 611 9876 633 1234 \N \N 2,4,5,6,10 -7 \N 9876 733 1234 300 300 1,2,4,6 - --- !seq_type_col_multi_rows_1 -- -0 0 0 0 0 0 0 -1 1 1 1 1 1 10 -2 2 2 2 2 2 20 -3 3 3 3 3 3 30 -4 4 4 4 4 4 40 -5 5 5 5 5 5 50 - --- !seq_type_col_multi_rows_2 -- -0 0 0 0 0 0 0 -1 888 888 1 1 1 20 3,4,5,6 -2 2 2 777 777 777 20 1,2,6,10 -3 3 3 3 3 3 30 -4 4 4 4 4 4 40 -5 5 5 5 5 5 50 -6 666 666 0 1234 \N 30 3,4,5,6 - --- !seq_type_col_multi_rows_3 -- -0 0 0 0 0 0 0 -1 888 888 1 1 1 20 3,4,5,6 -2 2 2 777 777 777 20 1,2,6,10 -3 777 777 555 555 555 53 6 -4 123 123 123 987 123 40 6,10 -5 5 5 5 5 5 50 -6 666 666 0 1234 \N 30 3,4,5,6 -7 666 666 0 1234 \N 30 3,4,5,6 - --- !seq_map_col_no_default_val_multi_rows_1 -- -0 0 0 0 0 0 0 0 -1 1 1 1 1 1 10 10 -2 2 2 2 2 2 20 20 -3 3 3 3 3 3 30 30 -4 4 4 4 4 4 40 40 -5 5 5 5 5 5 50 50 - --- !seq_map_col_no_default_val_multi_rows_2 -- -0 0 0 0 0 0 0 0 -1 888 888 1 1 1 20 20 3,4,5,7 -2 2 2 777 777 777 20 20 1,2,6,7,11 -3 3 3 3 3 3 30 30 -4 4 4 4 4 4 40 40 -5 5 5 5 5 5 50 50 -6 666 666 0 1234 \N 30 30 3,4,5,7 - --- !seq_map_col_no_default_val_multi_rows_3 -- -0 0 0 0 0 0 0 -1 888 888 1 1 1 20 3,4,5,7 -2 2 2 777 777 777 20 1,2,6,7,11 -3 777 777 555 555 555 53 7 -4 123 123 123 987 123 40 6,7,11 -5 5 5 5 5 5 50 -6 666 666 0 1234 \N 30 3,4,5,7 -7 666 666 0 1234 \N 70 3,4,5,7 - --- !seq_map_col_has_default_val_multi_rows_1 -- -0 0 0 0 0 0 0 0 -1 1 1 1 1 1 10 10 -2 2 2 2 2 2 20 20 -3 3 3 3 3 3 30 30 -4 4 4 4 4 4 40 40 -5 5 5 5 5 5 50 50 - --- !seq_map_col_has_default_val_multi_rows_2 -- -0 0 0 0 0 0 0 0 -1 888 888 1 1 1 20 20 3,4,5,7 -2 2 2 777 777 777 20 20 1,2,6,7,11 -3 3 3 3 3 3 30 30 -4 4 4 4 4 4 40 40 -5 5 5 5 5 5 50 50 -6 123 123 0 666 666 60 60 3,6,7,11 - --- !seq_map_col_has_default_val_multi_rows_3 -- -0 0 0 0 0 0 0 -1 888 888 1 1 1 20 3,4,5,7 -2 2 2 777 777 777 20 1,2,6,7,11 -3 777 777 555 555 555 53 7 -4 123 123 123 987 123 40 6,7,11 -5 5 5 5 5 5 50 -6 123 123 0 666 666 60 3,6,7,11 -7 666 666 0 1234 \N 70 3,4,5,7 +5 111222 1234 7890 5 1234 100 4,7 +6 123 123 7890 666 666 70 7 +7 123 123 7890 666 666 70 7 +8 123 456 23456 1234 999 80 4,6,7,10 +9 111222 1234 7890 5678 1234 100 7 -- !seq1 -- 1 256 1 1 256 @@ -333,25 +150,25 @@ -- !seq1 -- 1 256 9999 1 256 -2 256 2 -1 256 -3 3 9999 3 3 -4 4 9999 4 4 -5 255 \N -1 255 -6 266 \N -1 266 -7 \N 9999 \N \N +2 256 9999 -1 256 +3 3 9999 -1 3 +4 4 9999 -1 4 +5 255 9999 -1 255 +6 266 9999 -1 266 +7 \N 9999 -1 \N -- !seq1 -- 1 256 1 1 256 1 256 9999 1 256 2 255 2 2 255 -2 256 2 -1 256 +2 256 9999 -1 256 3 3 3 3 3 -3 3 9999 3 3 +3 3 9999 -1 3 4 4 4 4 4 -4 4 9999 4 4 -5 255 \N -1 255 -6 266 \N -1 266 -7 \N 9999 \N \N +4 4 9999 -1 4 +5 255 9999 -1 255 +6 266 9999 -1 266 +7 \N 9999 -1 \N -- !seq2 -- 1 256 1 1 256 @@ -361,23 +178,23 @@ -- !seq2 -- 1 256 9999 1 256 -2 256 2 -1 256 -3 3 9999 3 3 -4 4 9999 4 4 -5 256 9999 \N 256 -6 266 \N -1 266 -7 256 9999 \N 256 +2 256 9999 -1 256 +3 3 9999 -1 3 +4 4 9999 -1 4 +5 255 9999 -1 255 +6 266 9999 -1 266 +7 \N 9999 -1 \N -- !seq2 -- 1 256 1 1 256 1 256 9999 1 256 2 255 2 2 255 -2 256 2 -1 256 +2 256 9999 -1 256 3 3 3 3 3 -3 3 9999 3 3 +3 3 9999 -1 3 4 4 4 4 4 -4 4 9999 4 4 -5 256 9999 \N 256 -6 266 \N -1 266 -7 256 9999 \N 256 +4 4 9999 -1 4 +5 255 9999 -1 255 +6 266 9999 -1 266 +7 \N 9999 -1 \N diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.out b/regression-test/data/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.out new file mode 100644 index 00000000000000..6dc2e7e3b3a274 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 100 1 1 1 1 +2 100 2 2 2 2 +3 100 3 3 3 3 +4 100 4 4 4 4 + +-- !sql -- +1 \N 987 77777 1234 \N 100 +2 \N 987 77777 1234 \N 200 +3 100 987 77777 3 3 100 +4 \N 987 77777 1234 \N 100 + +-- !inspect -- +1 100 1 1 1 1 100 2 0 +1 100 1 1 1 1 100 3 1 +1 100 987 77777 1 1 100 5 0 +1 \N 987 77777 1234 \N 100 5 0 +2 100 2 2 2 2 100 2 0 +2 200 2 2 2 2 200 3 1 +2 100 987 77777 2 2 100 5 0 +2 \N 987 77777 1234 \N 200 5 0 +3 100 3 3 3 3 100 2 0 +3 50 \N 9876 1234 \N 50 3 1 +3 100 987 77777 3 3 100 5 0 +4 100 4 4 4 4 100 2 0 +4 100 4 4 4 4 100 4 1 +4 100 987 77777 4 4 100 5 0 +4 \N 987 77777 1234 \N 100 5 0 + diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.out new file mode 100644 index 00000000000000..18552473faa085 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1000 1 1 1 1 + +-- !sql1 -- + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 + +-- !sql2 -- +1 \N \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 +1 \N \N 9876 777 8888 1000 0 4 + +-- !sql3 -- +1 \N \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 +1 \N \N 9876 777 8888 1000 0 4 +1 999 -1 9876 -2 \N 999 0 5 + +-- !sql -- +1 1000 1 1 1 1 + +-- !sql1 -- + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 + +-- !sql2 -- +1 \N \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 +1 \N \N 9876 777 8888 1000 0 4 + +-- !sql3 -- +1 \N \N 9876 777 8888 1000 0 + +-- !inspect -- +1 1000 1 1 1 1 1000 0 2 +1 1000 1 1 1 1 1000 1 3 +1 \N \N 9876 777 8888 1000 0 4 +1 999 -1 9876 -2 \N 999 0 5 + diff --git a/regression-test/suites/fault_injection_p0/flexible/test_fleixble_partial_update_publish_conflict_seq.groovy b/regression-test/suites/fault_injection_p0/flexible/test_fleixble_partial_update_publish_conflict_seq.groovy deleted file mode 100644 index 49f28f88912309..00000000000000 --- a/regression-test/suites/fault_injection_p0/flexible/test_fleixble_partial_update_publish_conflict_seq.groovy +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.junit.Assert -import java.util.concurrent.TimeUnit -import org.awaitility.Awaitility - -suite("test_flexible_partial_update_publish_conflict_seq", "nonConcurrent") { - - def tableName = "test_flexible_partial_update_publish_conflict_seq" - sql """ DROP TABLE IF EXISTS ${tableName} force;""" - sql """ CREATE TABLE ${tableName} ( - `k` int(11) NULL, - `v1` BIGINT NULL, - `v2` BIGINT NULL, - `v3` BIGINT NULL, - `v4` BIGINT NULL, - ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 - PROPERTIES( - "replication_num" = "1", - "enable_unique_key_merge_on_write" = "true", - "light_schema_change" = "true", - "enable_unique_key_skip_bitmap_column" = "true", - "function_column.sequence_col" = "v1", - "store_row_column" = "false"); """ - def show_res = sql "show create table ${tableName}" - assertTrue(show_res.toString().contains('"enable_unique_key_skip_bitmap_column" = "true"')) - sql """insert into ${tableName} values(1,1,1,1,1),(2,2,2,2,2),(4,4,4,4,4),(5,5,5,5,5),(6,6,6,6,6);""" - order_qt_sql "select k,v1,v2,v3,v4,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" - - def beNodes = sql_return_maparray("show backends;") - def tabletStat = sql_return_maparray("show tablets from ${tableName};").get(0) - def tabletBackendId = tabletStat.BackendId - def tabletId = tabletStat.TabletId - def tabletBackend; - for (def be : beNodes) { - if (be.BackendId == tabletBackendId) { - tabletBackend = be - break; - } - } - logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); - - - def enable_publish_spin_wait = { - if (isCloudMode()) { - GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait") - } else { - GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait") - } - } - - def enable_block_in_publish = { - if (isCloudMode()) { - GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") - } else { - GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") - } - } - - def disable_block_in_publish = { - if (isCloudMode()) { - GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") - } else { - GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") - } - } - - def inspectRows = { sqlStr -> - sql "set skip_delete_sign=true;" - sql "set skip_delete_bitmap=true;" - sql "sync" - qt_inspect sqlStr - sql "set skip_delete_sign=false;" - sql "set skip_delete_bitmap=false;" - sql "sync" - } - - try { - GetDebugPoint().clearDebugPointsForAllFEs() - GetDebugPoint().clearDebugPointsForAllBEs() - - // block the flexible partial update in publish phase - enable_publish_spin_wait() - enable_block_in_publish() - def t1 = Thread.start { - streamLoad { - table "${tableName}" - set 'format', 'json' - set 'read_json_by_line', 'true' - set 'strict_mode', 'false' - set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' - file "test3.json" - time 20000 - } - } - - Thread.sleep(500) - - def t2 = Thread.start { - streamLoad { - table "${tableName}" - set 'format', 'json' - set 'read_json_by_line', 'true' - set 'strict_mode', 'false' - set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' - file "test4.json" - time 20000 - } - } - - Thread.sleep(500) - - disable_block_in_publish() - t1.join() - t2.join() - - qt_sql "select k,v1,v2,v3,v4,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} order by k;" - inspectRows "select k,v1,v2,v3,v4,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} order by k,__DORIS_VERSION_COL__,__DORIS_SEQUENCE_COL__;" - } catch(Exception e) { - logger.info(e.getMessage()) - throw e - } finally { - GetDebugPoint().clearDebugPointsForAllFEs() - GetDebugPoint().clearDebugPointsForAllBEs() - } - - // sql "DROP TABLE IF EXISTS ${tableName};" -} diff --git a/regression-test/suites/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.groovy b/regression-test/suites/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.groovy deleted file mode 100644 index 23532d446469e1..00000000000000 --- a/regression-test/suites/fault_injection_p0/flexible/test_flexible_partial_update_publish_conflict.groovy +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.junit.Assert - -suite("test_flexible_partial_update_publish_conflict", "nonConcurrent") { - - def tableName = "test_flexible_partial_update_publish_conflict" - sql """ DROP TABLE IF EXISTS ${tableName} """ - sql """ CREATE TABLE ${tableName} ( - `k` int(11) NULL, - `v1` BIGINT NULL, - `v2` BIGINT NULL DEFAULT "9876", - `v3` BIGINT NOT NULL, - `v4` BIGINT NOT NULL DEFAULT "1234", - `v5` BIGINT NULL - ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 - PROPERTIES( - "replication_num" = "1", - "enable_unique_key_merge_on_write" = "true", - "light_schema_change" = "true", - "enable_unique_key_skip_bitmap_column" = "true", - "store_row_column" = "false"); """ - def show_res = sql "show create table ${tableName}" - assertTrue(show_res.toString().contains('"enable_unique_key_skip_bitmap_column" = "true"')) - sql """insert into ${tableName} select number, number, number, number, number, number from numbers("number" = "6"); """ - order_qt_sql "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" - - def enable_publish_spin_wait = { - if (isCloudMode()) { - GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait") - } else { - GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait") - } - } - - def enable_block_in_publish = { - if (isCloudMode()) { - GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") - } else { - GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") - } - } - - def disable_block_in_publish = { - if (isCloudMode()) { - GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block") - } else { - GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.block") - } - } - - try { - GetDebugPoint().clearDebugPointsForAllFEs() - GetDebugPoint().clearDebugPointsForAllBEs() - - // block the partial update in publish phase - enable_publish_spin_wait() - enable_block_in_publish() - def t1 = Thread.start { - streamLoad { - table "${tableName}" - set 'format', 'json' - set 'read_json_by_line', 'true' - set 'strict_mode', 'false' - set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' - file "test1.json" - time 20000 - } - } - - Thread.sleep(500) - - def t2 = Thread.start { - streamLoad { - table "${tableName}" - set 'format', 'json' - set 'read_json_by_line', 'true' - set 'strict_mode', 'false' - set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' - file "test2.json" - time 20000 - } - } - - Thread.sleep(500) - - disable_block_in_publish() - t1.join() - t2.join() - - order_qt_sql "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" - - } catch(Exception e) { - logger.info(e.getMessage()) - throw e - } finally { - GetDebugPoint().clearDebugPointsForAllFEs() - GetDebugPoint().clearDebugPointsForAllBEs() - } - - sql "DROP TABLE IF EXISTS ${tableName};" -} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.groovy new file mode 100644 index 00000000000000..fb4f1b10a17cfb --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_auto_inc_replica_consistency.groovy @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +suite("test_auto_inc_replica_consistency") { + if (isCloudMode()) { + logger.info("skip test_auto_inc_replica_consistency in cloud mode") + return + } + + def dbName = context.config.getDbNameByFile(context.file) + def tableName = "test_auto_inc_replica_consistency" + sql """ DROP TABLE IF EXISTS ${tableName} FORCE;""" + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `id` BIGINT NOT NULL AUTO_INCREMENT(20) + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "store_row_column" = "false"); """ + + def beNodes = sql_return_maparray("show backends;") + if (beNodes.size() > 1) { + logger.info("skip to run the case when there are more than 1 BE.") + return + } + + def do_streamload_2pc_commit = { txnId -> + def command = "curl -X PUT --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}" + + " -H txn_id:${txnId}" + + " -H txn_operation:commit" + + " http://${context.config.feHttpAddress}/api/${dbName}/${tableName}/_stream_load_2pc" + log.info("http_stream execute 2pc: ${command}") + + def process = command.execute() + def code = process.waitFor() + def out = process.text + def json2pc = parseJson(out) + log.info("http_stream 2pc result: ${out}".toString()) + assertEquals(code, 0) + assertEquals("success", json2pc.status.toLowerCase()) + } + + def wait_for_publish = {txnId, waitSecond -> + String st = "PREPARE" + while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && waitSecond > 0) { + Thread.sleep(1000) + waitSecond -= 1 + def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" + assertNotNull(result) + st = result[0].TransactionStatus + } + log.info("Stream load with txn ${txnId} is ${st}") + assertEquals(st, "VISIBLE") + } + + + def txnId1, txnId2 + String load1 = """{"k":1,"v1":100}""" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load1.getBytes()) + time 60000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + txnId1 = json.TxnId + assert "success" == json.Status.toLowerCase() + } + } + + String load2 = """{"k":1,"v2":200}""" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load2.getBytes()) + time 60000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + txnId2 = json.TxnId + assert "success" == json.Status.toLowerCase() + } + } + + do_streamload_2pc_commit(txnId1) + wait_for_publish(txnId1, 10) + do_streamload_2pc_commit(txnId2) + wait_for_publish(txnId2, 10) + + sql "insert into ${tableName}(k,v1,v2) values(2,2,2);" + + qt_sql "select k,v1,v2,id from ${tableName} order by k;" + sql "set skip_delete_bitmap=true;" + sql "sync;" + qt_sql "select k,v1,v2,id,__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__;" +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_auto_inc_compaction.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_auto_inc_compaction.groovy new file mode 100644 index 00000000000000..6079d40fa56b93 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_auto_inc_compaction.groovy @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +suite("test_f_auto_inc_compaction") { + if (isCloudMode()) { + logger.info("skip test_f_auto_inc_compaction in cloud mode") + return + } + + def dbName = context.config.getDbNameByFile(context.file) + def tableName = "test_f_auto_inc_compaction" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `id` BIGINT NOT NULL AUTO_INCREMENT + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "disable_auto_compaction" = "true", + "store_row_column" = "false"); """ + + sql """insert into ${tableName}(k,v1,v2) values(1,1,1),(2,2,2),(3,3,3),(4,4,4),(5,5,5),(6,6,6);""" + qt_sql "select k,v1,v2,id from ${tableName} order by k;" + sql "delete from ${tableName} where k=1;" + sql "delete from ${tableName} where k=3;" + sql "delete from ${tableName} where k=6;" + qt_sql "select k,v1,v2 from ${tableName} order by k;" + qt_check_auto_inc_dup "select count(*) from ${tableName} group by id having count(*) > 1;" + + def beNodes = sql_return_maparray("show backends;") + def tabletStat = sql_return_maparray("show tablets from ${tableName};").get(0) + def tabletBackendId = tabletStat.BackendId + def tabletId = tabletStat.TabletId + def tabletBackend; + for (def be : beNodes) { + if (be.BackendId == tabletBackendId) { + tabletBackend = be + break; + } + } + logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); + + def check_rs_metas = { expected_rs_meta_size, check_func -> + if (isCloudMode()) { + return + } + + def metaUrl = sql_return_maparray("show tablets from ${tableName};").get(0).MetaUrl + def (code, out, err) = curl("GET", metaUrl) + Assert.assertEquals(code, 0) + def jsonMeta = parseJson(out.trim()) + + logger.info("jsonMeta.rs_metas.size(): ${jsonMeta.rs_metas.size()}, expected_rs_meta_size: ${expected_rs_meta_size}") + Assert.assertEquals(jsonMeta.rs_metas.size(), expected_rs_meta_size) + for (def meta : jsonMeta.rs_metas) { + int startVersion = meta.start_version + int endVersion = meta.end_version + int numSegments = meta.num_segments + int numRows = meta.num_rows + String overlapPb = meta.segments_overlap_pb + logger.info("[${startVersion}-${endVersion}] ${overlapPb} ${meta.num_segments} ${numRows} ${meta.rowset_id_v2}") + check_func(startVersion, endVersion, numSegments, numRows, overlapPb) + } + } + + check_rs_metas(5, {int startVersion, int endVersion, int numSegments, int numRows, String overlapPb -> + if (startVersion == 0) { + // [0-1] + Assert.assertEquals(endVersion, 1) + Assert.assertEquals(numSegments, 0) + } else { + // [2-2], [3-3], [4-4], [5-5] + Assert.assertEquals(startVersion, endVersion) + Assert.assertEquals(numSegments, 1) + } + }) + + def do_streamload_2pc_commit = { txnId -> + def command = "curl -X PUT --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}" + + " -H txn_id:${txnId}" + + " -H txn_operation:commit" + + " http://${context.config.feHttpAddress}/api/${dbName}/${tableName}/_stream_load_2pc" + log.info("http_stream execute 2pc: ${command}") + + def process = command.execute() + def code = process.waitFor() + def out = process.text + def json2pc = parseJson(out) + log.info("http_stream 2pc result: ${out}".toString()) + assertEquals(code, 0) + assertEquals("success", json2pc.status.toLowerCase()) + } + + def wait_for_publish = {txnId, waitSecond -> + String st = "PREPARE" + while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && waitSecond > 0) { + Thread.sleep(1000) + waitSecond -= 1 + def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" + assertNotNull(result) + st = result[0].TransactionStatus + } + log.info("Stream load with txn ${txnId} is ${st}") + assertEquals(st, "VISIBLE") + } + + + def txnId1, txnId2 + String load1 = """99,99,99,99""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'two_phase_commit', 'true' + inputStream new ByteArrayInputStream(load1.getBytes()) + time 60000 // limit inflight 60s + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + txnId1 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + + String load2 = + """{"k":3,"v1":100} + {"k":6,"v2":600}""" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load2.getBytes()) + time 60000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + txnId2 = json.TxnId + assert "success" == json.Status.toLowerCase() + } + } + + + // most have a new load after load2's max version sees in flush phase. Otherwise load2 will skip to calc delete bitmap + // of rowsets which is produced by compaction and whose end version is lower than the max version + // the load1 sees in flush phase, see https://github.com/apache/doris/pull/38487 for details + + // let load1 publish + do_streamload_2pc_commit(txnId1) + wait_for_publish(txnId1, 10) + Thread.sleep(1000) + + + trigger_and_wait_compaction(tableName, "full") + + check_rs_metas(1, {int startVersion, int endVersion, int numSegments, int numRows, String overlapPb -> + // check the rowset produced by full compaction + // [0-6] + Assert.assertEquals(startVersion, 0) + Assert.assertEquals(endVersion, 6) + Assert.assertEquals(numRows, 4) + Assert.assertEquals(overlapPb, "NONOVERLAPPING") + }) + + qt_after_compaction "select k,v1,v2 from ${tableName} order by k;" + qt_check_auto_inc_dup "select count(*) from ${tableName} group by id having count(*) > 1;" + + do_streamload_2pc_commit(txnId2) + wait_for_publish(txnId2, 10) + + qt_sql "select k,v1,v2 from ${tableName} order by k;" + qt_check_auto_inc_dup "select count(*) from ${tableName} group by id having count(*) > 1;" + qt_check_auto_inc_val "select count(*) from ${tableName} where id=0;" +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.groovy new file mode 100644 index 00000000000000..0c62427f94bd41 --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_f_seq_publish_read_from_old.groovy @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert + +suite("test_f_seq_publish_read_from_old") { + if (isCloudMode()) { + logger.info("skip test_f_seq_publish_read_from_old in cloud mode") + return + } + + def dbName = context.config.getDbNameByFile(context.file) + def tableName = "test_f_seq_publish_read_from_old" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `v3` BIGINT NOT NULL DEFAULT "9876", + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "function_column.sequence_col" = "v1", + "store_row_column" = "false"); """ + sql """insert into ${tableName} values(1,100,1,1,1,1),(2,100,2,2,2,2),(3,100,3,3,3,3),(4,100,4,4,4,4);""" + qt_sql "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + + def inspectRows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + + def do_streamload_2pc_commit = { txnId -> + def command = "curl -X PUT --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}" + + " -H txn_id:${txnId}" + + " -H txn_operation:commit" + + " http://${context.config.feHttpAddress}/api/${dbName}/${tableName}/_stream_load_2pc" + log.info("http_stream execute 2pc: ${command}") + + def process = command.execute() + def code = process.waitFor() + def out = process.text + def json2pc = parseJson(out) + log.info("http_stream 2pc result: ${out}".toString()) + assertEquals(code, 0) + assertEquals("success", json2pc.status.toLowerCase()) + } + + def wait_for_publish = {txnId, waitSecond -> + String st = "PREPARE" + while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && waitSecond > 0) { + Thread.sleep(1000) + waitSecond -= 1 + def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" + assertNotNull(result) + st = result[0].TransactionStatus + } + log.info("Stream load with txn ${txnId} is ${st}") + assertEquals(st, "VISIBLE") + } + + def txnId1, txnId2 + String load1 = + """{"k":1,"v1":100,"__DORIS_DELETE_SIGN__":1} + {"k":2,"v1":200,"__DORIS_DELETE_SIGN__":1} + {"k":3,"v1":50,"__DORIS_DELETE_SIGN__":1} + {"k":4,"__DORIS_DELETE_SIGN__":1}""" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load1.getBytes()) + time 60000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + txnId1 = json.TxnId + assert "success" == json.Status.toLowerCase() + } + } + + String load2 = + """{"k":1,"v2":987,"v3":77777} + {"k":2,"v2":987,"v3":77777} + {"k":3,"v2":987,"v3":77777} + {"k":4,"v2":987,"v3":77777}""" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load2.getBytes()) + time 60000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + txnId2 = json.TxnId + assert "success" == json.Status.toLowerCase() + } + } + do_streamload_2pc_commit(txnId1) + wait_for_publish(txnId1, 10) + + do_streamload_2pc_commit(txnId2) + wait_for_publish(txnId2, 10) + + sql "sync;" + qt_sql "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + inspectRows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_VERSION_COL__,__DORIS_SEQUENCE_COL__;" +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.groovy new file mode 100644 index 00000000000000..7f257a2c336a1b --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_fleixble_partial_update_publish_conflict_seq.groovy @@ -0,0 +1,236 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +suite("test_flexible_partial_update_publish_conflict_seq") { + if (isCloudMode()) { + logger.info("skip test_flexible_partial_update_publish_conflict_seq in cloud mode") + return + } + def dbName = context.config.getDbNameByFile(context.file) + def tableName = "test_flexible_partial_update_publish_conflict_seq" + sql """ DROP TABLE IF EXISTS ${tableName} force;""" + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `v3` BIGINT NULL, + `v4` BIGINT NULL, + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "function_column.sequence_col" = "v1", + "store_row_column" = "false"); """ + def show_res = sql "show create table ${tableName}" + assertTrue(show_res.toString().contains('"enable_unique_key_skip_bitmap_column" = "true"')) + sql """insert into ${tableName} values(1,1,1,1,1),(2,1,2,2,2),(3,1,3,3,3),(4,1,4,4,4),(14,1,14,14,14),(15,1,15,15,15),(16,1,16,16,16),(17,1,17,17,17),(27,1,27,27,27),(28,1,28,28,28),(29,1,29,29,29);""" + qt_sql "select k,v1,v2,v3,v4,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} order by k;" + + def inspectRows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + + def do_streamload_2pc_commit = { txnId -> + def command = "curl -X PUT --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}" + + " -H txn_id:${txnId}" + + " -H txn_operation:commit" + + " http://${context.config.feHttpAddress}/api/${dbName}/${tableName}/_stream_load_2pc" + log.info("http_stream execute 2pc: ${command}") + + def process = command.execute() + def code = process.waitFor() + def out = process.text + def json2pc = parseJson(out) + log.info("http_stream 2pc result: ${out}".toString()) + assertEquals(code, 0) + assertEquals("success", json2pc.status.toLowerCase()) + } + + def wait_for_publish = {txnId, waitSecond -> + String st = "PREPARE" + while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && waitSecond > 0) { + Thread.sleep(1000) + waitSecond -= 1 + def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" + assertNotNull(result) + st = result[0].TransactionStatus + } + log.info("Stream load with txn ${txnId} is ${st}") + assertEquals(st, "VISIBLE") + } + + def txnId1, txnId2 + // load 1 + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "test3.json" + time 60000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + txnId1 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + // load 2 + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "test4.json" + time 60000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + txnId2 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + // row(1)'s seq value: origin < load 2 < load 1 + // row(2)'s seq value: origin < load 1 < load 2 + // row(3)'s seq value: origin < load 1, load 2 without seq val + // row(4)'s seq value: origin < load 2, load 1 without seq val + + // row(14)'s seq value: origin < load 2(with delete sign) < load 1 + // row(15)'s seq value: origin < load 1 < load 2(with delete sign) + // row(16)'s seq value: origin < load 1, load 2 without seq val, with delete sign + // row(17)'s seq value: origin < load 2(with delete sign), load 1 without seq val + + // row(27)'s seq value: origin < load 2 < load 1(with delete sign) + // row(28)'s seq value: origin < load 1(with delete sign) < load 2 + // row(29)'s seq value: origin < load 1(with delete sign), load 2 without seq val + // row(30)'s seq value: origin < load 2, load 1 without seq val, with delete sign + + do_streamload_2pc_commit(txnId1) + wait_for_publish(txnId1, 60) + do_streamload_2pc_commit(txnId2) + wait_for_publish(txnId2, 60) + + qt_sql "select k,v1,v2,v3,v4,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__ from ${tableName} order by k;" + + sql "set skip_delete_sign=true;" + qt_skip_delete_sign "select k,v1,v2,v3,v4,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__ from ${tableName} order by k;" + sql "set skip_delete_sign=false;" + + inspectRows "select k,v1,v2,v3,v4,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_VERSION_COL__;" + + + // =========================================================================================== + // publish alignment read from rowsets which have multi-segments + sql "truncate table ${tableName}" + def txnId3, txnId4, txnId5 + + String load3 = """1,10,1,1,1 +2,20,2,2,2""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'two_phase_commit', 'true' + inputStream new ByteArrayInputStream(load3.getBytes()) + time 60000 // limit inflight 60s + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId3 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + String load4 = """1,99,99,99""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,v1,v2,v3' + set 'strict_mode', "false" + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FIXED_COLUMNS' + inputStream new ByteArrayInputStream(load4.getBytes()) + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId4 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "test6.json" + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId5 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + // let t3 and t4 publish + do_streamload_2pc_commit(txnId3) + wait_for_publish(txnId3, 60) + do_streamload_2pc_commit(txnId4) + wait_for_publish(txnId4, 60) + qt_sql1 "select k,v1,v2,v3,v4 from ${tableName} order by k;" + + do_streamload_2pc_commit(txnId5) + wait_for_publish(txnId5, 60) + qt_sql2 "select k,v1,v2,v3,v4 from ${tableName} order by k;" + inspectRows "select k,v1,v2,v3,v4,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_VERSION_COL__,__DORIS_SEQUENCE_COL__;" + +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/publish/test_flexible_partial_update_publish_conflict.groovy b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_flexible_partial_update_publish_conflict.groovy new file mode 100644 index 00000000000000..0101e775f7562d --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/flexible/publish/test_flexible_partial_update_publish_conflict.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert + +suite("test_flexible_partial_update_publish_conflict") { + if (isCloudMode()) { + logger.info("skip test_flexible_partial_update_publish_conflict in cloud mode") + return + } + def dbName = context.config.getDbNameByFile(context.file) + def tableName = "test_flexible_partial_update_publish_conflict" + sql """ DROP TABLE IF EXISTS ${tableName} FORCE;""" + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL DEFAULT "9876", + `v3` BIGINT NOT NULL, + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "store_row_column" = "false"); """ + def show_res = sql "show create table ${tableName}" + assertTrue(show_res.toString().contains('"enable_unique_key_skip_bitmap_column" = "true"')) + sql """insert into ${tableName} select number, number, number, number, number, number from numbers("number" = "6"); """ + order_qt_sql "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" + + def do_streamload_2pc_commit = { txnId -> + def command = "curl -X PUT --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}" + + " -H txn_id:${txnId}" + + " -H txn_operation:commit" + + " http://${context.config.feHttpAddress}/api/${dbName}/${tableName}/_stream_load_2pc" + log.info("http_stream execute 2pc: ${command}") + + def process = command.execute() + def code = process.waitFor() + def out = process.text + def json2pc = parseJson(out) + log.info("http_stream 2pc result: ${out}".toString()) + assertEquals(code, 0) + assertEquals("success", json2pc.status.toLowerCase()) + } + + def wait_for_publish = {txnId, waitSecond -> + String st = "PREPARE" + while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && waitSecond > 0) { + Thread.sleep(1000) + waitSecond -= 1 + def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" + assertNotNull(result) + st = result[0].TransactionStatus + } + log.info("Stream load with txn ${txnId} is ${st}") + assertEquals(st, "VISIBLE") + } + + // block the partial update in publish phase + def txnId1, txnId2 + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "test1.json" + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId1 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "test2.json" + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId2 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + do_streamload_2pc_commit(txnId1) + wait_for_publish(txnId1, 60) + do_streamload_2pc_commit(txnId2) + wait_for_publish(txnId2, 60) + + order_qt_sql "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" + + + // ================================================================================================== + // publish alignment read from rowsets which have multi-segments + sql "truncate table ${tableName}" + + def txnId3, txnId4, txnId5 + + String load3 = """1,1,1,1,1,1 +2,2,2,2,2,2""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'two_phase_commit', 'true' + inputStream new ByteArrayInputStream(load3.getBytes()) + time 60000 // limit inflight 60s + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId3 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + String load4 = """1,99,99,99""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,v1,v2,v3' + set 'strict_mode', "false" + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FIXED_COLUMNS' + inputStream new ByteArrayInputStream(load4.getBytes()) + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId4 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "test5.json" + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId5 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + // let t3 and t4 publish + do_streamload_2pc_commit(txnId3) + wait_for_publish(txnId3, 60) + do_streamload_2pc_commit(txnId4) + wait_for_publish(txnId4, 60) + qt_sql1 "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + + do_streamload_2pc_commit(txnId5) + wait_for_publish(txnId5, 60) + qt_sql2 "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" +} diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_f_seq_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_f_seq_read_from_old.groovy new file mode 100644 index 00000000000000..4eed4ada2bd48a --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_f_seq_read_from_old.groovy @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_f_seq_read_from_old') { + def inspect_rows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + for (def use_row_store : [false, true]) { + logger.info("current params: use_row_store: ${use_row_store}") + def tableName = "test_f_seq_read_from_old_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `v3` BIGINT NOT NULL DEFAULT "9876", + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "function_column.sequence_col" = "v1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName} values(1,1000,1,1,1,1);" + qt_sql "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + + + String load1 = """ {"k":1,"__DORIS_DELETE_SIGN__":1} """ + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load1.getBytes()) + time 20000 + } + qt_sql1 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} order by k,__DORIS_VERSION_COL__;" + + String load2 = """ {"k":1,"v4":777,"v5":8888} """ + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load2.getBytes()) + time 20000 + } + qt_sql2 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} order by k,__DORIS_VERSION_COL__;" + + String load3 = """ {"k":1,"v1":999,"v2":-1,"v4":-2} """ + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + inputStream new ByteArrayInputStream(load3.getBytes()) + time 20000 + } + qt_sql3 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} order by k,__DORIS_VERSION_COL__;" + } +} \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy index 93e56b8a85b00e..bd9f69fc3ddc90 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_auto_inc.groovy @@ -92,5 +92,30 @@ suite('test_flexible_partial_update_auto_inc') { qt_autoinc_val_2 "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} where k not in (8,10) order by k;" qt_autoinc_val_3 "select k,v1,v2,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName} where k in (8,10) order by k;" qt_autoinc_val_4 "select count(distinct v3) from ${tableName} where k in (8,10);" + + def originAutoIncVals = [] + def res1 = sql_return_maparray "select k,v1,v2,v3,v4,v5 from ${tableName} where k in (8,10)"; + for (def row : res1) { + originAutoIncVals << row.v3 + } + Collections.sort(originAutoIncVals) + logger.info("originAutoIncVals; ${originAutoIncVals}") + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "autoinc2.json" + time 20000 + } + def autoIncVals = [] + def res2 = sql_return_maparray "select k,v1,v2,v3,v4,v5 from ${tableName} where k in (8,10)"; + for (def row : res1) { + autoIncVals << row.v3 + } + Collections.sort(autoIncVals) + logger.info("autoIncVals: ${autoIncVals}") + assertEquals(originAutoIncVals, autoIncVals) } } \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_default_value.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_default_value.groovy new file mode 100644 index 00000000000000..49259121956b9e --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_default_value.groovy @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_flexible_partial_update_default_value') { + + for (def use_row_store : [false, true]) { + logger.info("current params: use_row_store: ${use_row_store}") + def tableName = "test_f_default_value_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL DEFAULT "9876", + `v3` varchar(100) NOT NULL default "test", + `v4` BIGINT DEFAULT null, + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "store_row_column" = "${use_row_store}"); """ + + sql """insert into ${tableName} select number, number, number, number, number from numbers("number" = "4"); """ + qt_sql "select k,v1,v2,v3,v4 from ${tableName} order by k;" + + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "default1.json" + time 20000 + } + qt_sql "select k,v1,v2,v3,v4 from ${tableName} order by k;" + + tableName = "test_f_default_value2_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `t1` datetime default current_timestamp, + `t2` datetime(6) default current_timestamp(6), + `t3` DATE DEFAULT CURRENT_DATE + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "store_row_column" = "${use_row_store}"); """ + + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "default2.json" + time 20000 + } + qt_sql "select k,v1,v2 from ${tableName} where t1 > '2024-10-28 00:00:00' order by k;" + qt_sql "select k,v1,v2 from ${tableName} where t2 > '2024-10-28 00:00:00' order by k;" + qt_sql "select k,v1,v2 from ${tableName} where t3 > '2024-10-28 00:00:00' order by k;" + // these generated default value should be the same in one batch + qt_sql "select count(distinct t1) from ${tableName};" + qt_sql "select count(distinct t2) from ${tableName};" + qt_sql "select count(distinct t3) from ${tableName};" + + + tableName = "test_f_default_value3_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `t1` datetime default current_timestamp on update current_timestamp, + `t2` datetime(6) default current_timestamp(6) on update current_timestamp(6), + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "store_row_column" = "${use_row_store}"); """ + sql """insert into ${tableName} select number, number, number, '2020-01-01 00:00:00', '2020-01-01 00:00:00' from numbers("number" = "5"); """ + qt_sql "select k,v1,v2,t1,t2 from ${tableName} order by k;" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "default3.json" + time 20000 + } + qt_sql "select k,v1,v2 from ${tableName} order by k;" + qt_sql "select k,v1,v2 from ${tableName} where t1 > '2024-10-28 00:00:00' order by k;" + qt_sql "select k,v1,v2 from ${tableName} where t2 > '2024-10-28 00:00:00' order by k;" + qt_sql "select k,v1,v2 from ${tableName} where t1 > '2024-10-28 00:00:00' and t2 > '2024-10-28 00:00:00' order by k;" + } +} \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy index 10fe0d1cdd1194..3fad13d5304985 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_delete_sign.groovy @@ -17,8 +17,18 @@ suite('test_flexible_partial_update_delete_sign') { - for (def use_row_store : [false, true]) { - logger.info("current params: use_row_store: ${use_row_store}") + def inspect_rows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + + for (def use_row_store : [false]) { + // logger.info("current params: use_row_store: ${use_row_store}") // 1. table without sequence col def tableName = "test_flexible_partial_update_delete_sign_${use_row_store}" @@ -39,17 +49,7 @@ suite('test_flexible_partial_update_delete_sign') { "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName} select number, number, number, number, number, number from numbers("number" = "6"); """ - order_qt_no_seq_col_1 "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" - - def inspect_rows = { sqlStr -> - sql "set skip_delete_sign=true;" - sql "set skip_delete_bitmap=true;" - sql "sync" - order_qt_inspect sqlStr - sql "set skip_delete_sign=false;" - sql "set skip_delete_bitmap=false;" - sql "sync" - } + order_qt_no_seq_col_1 "select k,v1,v2,v3,v4,v5 from ${tableName};" // update rows(2,4,5), delete rows(1,3), insert new rows(6), delete new rows(7) // should not fail although rows(1,3,7) doesn't specify v3(which is not nullable and has no default value) @@ -63,10 +63,10 @@ suite('test_flexible_partial_update_delete_sign') { file "delete1.json" time 20000 } - order_qt_no_seq_col_2 "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" - inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" + order_qt_no_seq_col_2 "select k,v1,v2,v3,v4,v5 from ${tableName};" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" - // delete rows(1,5) which have been deleted by delete sign before + // delete rows(1,7) which have been deleted by delete sign before streamLoad { table "${tableName}" set 'format', 'json' @@ -76,8 +76,8 @@ suite('test_flexible_partial_update_delete_sign') { file "delete2.json" time 20000 } - order_qt_no_seq_col_3 "select k,v1,v2,v3,v4,v5,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" - inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" + order_qt_no_seq_col_3 "select k,v1,v2,v3,v4,v5 from ${tableName};" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" // 2. table with sequence map col @@ -100,7 +100,7 @@ suite('test_flexible_partial_update_delete_sign') { "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName} select number, number, number, number, number, number from numbers("number" = "6"); """ - order_qt_seq_map_col_1 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" + order_qt_seq_map_col_1 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName};" // update rows(2,4,5), delete rows(1,3), insert new rows(6), delete new rows(7) // __DORIS_SEQUENCE_COL__ should be filled from old rows for rows(1,3) streamLoad { @@ -112,7 +112,245 @@ suite('test_flexible_partial_update_delete_sign') { file "delete1.json" time 20000 } - order_qt_seq_map_col_2 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" - inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" + order_qt_seq_map_col_2 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__ from ${tableName};" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + // ============================================================================================================================== + // 3. test insert after delete in one load + // 3.1 without seqeunce column + tableName = "test_flexible_partial_update_delete_sign3_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL DEFAULT "9876", + `v3` BIGINT NOT NULL, + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "store_row_column" = "${use_row_store}"); """ + + sql """insert into ${tableName} select number, number, number, number, number, number from numbers("number" = "10"); """ + qt_insert_after_delete_3_1 "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete3.json" + time 20000 + } + qt_insert_after_delete_3_1 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + tableName = "test_flexible_partial_update_delete_sign4_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL DEFAULT "9876", + `v3` BIGINT NOT NULL, + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "store_row_column" = "${use_row_store}"); """ + + sql """insert into ${tableName} select number, number, number, number, number, number from numbers("number" = "10"); """ + qt_insert_after_delete_3_1 "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete4.json" + time 20000 + } + qt_insert_after_delete_3_1 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + // 3.2 with sequence type column + tableName = "test_flexible_partial_update_delete_sign5_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} force;""" + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL DEFAULT "9876", + `v3` BIGINT NOT NULL DEFAULT "5432", + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL DEFAULT "9753" + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "function_column.sequence_type" = "int", + "store_row_column" = "${use_row_store}"); """ + sql """insert into ${tableName}(k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__) select number, number, number, number, number, number, null from numbers("number" = "15"); """ + qt_insert_after_delete_3_2_1 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName} order by k;" + // rows(1,2,3,4,5,6) are all without sequence column + // rows(7,8,9,10,11,12) are all with sequence column, seq col value is increasing + // row(1,7): delete + insert + // row(2,8): insert + delete + // row(3,9): delete + insert + delete + // row(4,10): insert + delete + insert + // row(5,11): delete + insert + delete + insert + // row(6,12): insert + delete + insert + delete + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete5.json" + time 20000 + } + qt_insert_after_delete_3_2_2 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + sql "truncate table ${tableName};" + sql """insert into ${tableName}(k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__) select number, number, number, number, number, number, null from numbers("number" = "15"); """ + qt_insert_after_delete_3_2_3 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName} order by k;" + // rows(7,8,9,10,11,12) same as above, insert some rows with lower seq value + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete6.json" + time 20000 + } + qt_insert_after_delete_3_2_3 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + sql "truncate table ${tableName};" + sql """insert into ${tableName}(k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__) select number, number, number, number, number, number, number*10 from numbers("number" = "13"); """ + qt_insert_after_delete_3_2_4 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName} order by k;" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete7.json" + time 20000 + } + qt_insert_after_delete_3_2_4 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + // 3.3 with sequence map col (no default value) + tableName = "test_flexible_partial_update_delete_sign6_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} force;""" + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL DEFAULT "9876", + `v3` BIGINT NOT NULL DEFAULT "5432", + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL DEFAULT "9753" + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "function_column.sequence_col" = "v1", + "store_row_column" = "${use_row_store}"); """ + sql """insert into ${tableName}(k,v1,v2,v3,v4,v5) select number, null, number, number, number, number from numbers("number" = "15"); """ + qt_insert_after_delete_3_3_1 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName} order by k;" + // rows(1,2,3,4,5,6) are all without sequence column + // rows(7,8,9,10,11,12) are all with sequence column, seq col value is increasing + // row(1,7): delete + insert + // row(2,8): insert + delete + // row(3,9): delete + insert + delete + // row(4,10): insert + delete + insert + // row(5,11): delete + insert + delete + insert + // row(6,12): insert + delete + insert + delete + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete8.json" + time 20000 + } + qt_insert_after_delete_3_3_2 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + sql "truncate table ${tableName};" + sql """insert into ${tableName}(k,v1,v2,v3,v4,v5) select number, null, number, number, number, number from numbers("number" = "15"); """ + qt_insert_after_delete_3_3_3 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName} order by k;" + // rows(7,8,9,10,11,12) same as above, insert some rows with lower seq value + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete9.json" + time 20000 + } + qt_insert_after_delete_3_3_4 "select k,v1,v2,v3,v4,v5,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + sql "truncate table ${tableName};" + sql """insert into ${tableName}(k,v1,v2,v3,v4,v5) select number, number*10, number, number, number, number from numbers("number" = "13"); """ + qt_insert_after_delete_3_3_5 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName} order by k;" + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete10.json" + time 20000 + } + qt_insert_after_delete_3_3_6 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" + + + // 3.4 with seq map col (has default value) + tableName = "test_flexible_partial_update_delete_sign6_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} force;""" + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL default "30", + `v2` BIGINT NULL DEFAULT "9876", + `v3` BIGINT NOT NULL DEFAULT "5432", + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL DEFAULT "9753" + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "function_column.sequence_col" = "v1", + "store_row_column" = "${use_row_store}"); """ + streamLoad { + table "${tableName}" + set 'format', 'json' + set 'read_json_by_line', 'true' + set 'strict_mode', 'false' + set 'unique_key_update_mode', 'UPDATE_FLEXIBLE_COLUMNS' + file "delete11.json" + time 20000 + } + qt_insert_after_delete_3_4 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__),__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__,v1,v2,v3,v4,v5;" } } \ No newline at end of file diff --git a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy index 79f5437d869dbd..8296531ea9f68c 100644 --- a/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy +++ b/regression-test/suites/unique_with_mow_p0/flexible/test_flexible_partial_update_seq_col.groovy @@ -17,7 +17,7 @@ suite('test_flexible_partial_update_seq_col') { - for (def use_row_store : [false, true]) { + for (def use_row_store : [false]) { logger.info("current params: use_row_store: ${use_row_store}") // 1.1. sequence map col(without default value) @@ -137,7 +137,8 @@ suite('test_flexible_partial_update_seq_col') { // ============================================================================================================================== // the below cases will have many rows with same keys in one load. Among rows with the same keys, some of them specify sequence col(sequence map col), - // some of them don't. Those + // some of them don't. + // The behavior should be the same as if these rows are inserted one by one // 2.1. sequence type col tableName = "test_flexible_partial_update_seq_type_col2_${use_row_store}" @@ -236,7 +237,7 @@ suite('test_flexible_partial_update_seq_col') { `v3` BIGINT NOT NULL, `v4` BIGINT NOT NULL DEFAULT "1234", `v5` BIGINT NULL, - `v6` BIGINT NULL default "60" + `v6` BIGINT NULL default "80" ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", @@ -249,9 +250,6 @@ suite('test_flexible_partial_update_seq_col') { order_qt_seq_map_col_has_default_val_multi_rows_1 "select k,v1,v2,v3,v4,v5,v6,__DORIS_SEQUENCE_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" // 2.3.1. rows with same keys are neighbers - // after merge in memtable, newly inserted rows(key=6) will has two rows, one with sequence map col value=30 - // one without sequence map value. Because the default value of sequence map col(`v6`) is 60, larger than 30, - // so the row with sequence map col will be deleted by the row without sequence map col streamLoad { table "${tableName}" set 'format', 'json' @@ -263,9 +261,6 @@ suite('test_flexible_partial_update_seq_col') { } order_qt_seq_map_col_has_default_val_multi_rows_2 "select k,v1,v2,v3,v4,v5,v6,__DORIS_SEQUENCE_COL__,BITMAP_TO_STRING(__DORIS_SKIP_BITMAP_COL__) from ${tableName};" // 2.3.2. rows with same keys are interleaved - // after merge in memtable, newly inserted rows(key=7) will has two rows, one with sequence map col value=70 - // one without sequence map value. Because the default value of sequence map col(`v6`) is 60, smaller than 70, - // so the row without sequence map col will be deleted by the row with sequence map col streamLoad { table "${tableName}" set 'format', 'json' diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.groovy new file mode 100644 index 00000000000000..81423436c069bd --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_p_seq_publish_read_from_old.groovy @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert + +suite("test_p_seq_publish_read_from_old") { + if (isCloudMode()) { + logger.info("skip test_p_seq_publish_read_from_old in cloud mode") + return + } + def dbName = context.config.getDbNameByFile(context.file) + def tableName = "test_p_seq_publish_read_from_old" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `v3` BIGINT NOT NULL DEFAULT "9876", + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "function_column.sequence_col" = "v1", + "store_row_column" = "false"); """ + sql """insert into ${tableName} values(1,100,1,1,1,1),(2,100,2,2,2,2),(3,100,3,3,3,3),(4,100,4,4,4,4);""" + qt_sql "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + + def inspectRows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + + def do_streamload_2pc_commit = { txnId -> + def command = "curl -X PUT --location-trusted -u ${context.config.feHttpUser}:${context.config.feHttpPassword}" + + " -H txn_id:${txnId}" + + " -H txn_operation:commit" + + " http://${context.config.feHttpAddress}/api/${dbName}/${tableName}/_stream_load_2pc" + log.info("http_stream execute 2pc: ${command}") + + def process = command.execute() + def code = process.waitFor() + def out = process.text + def json2pc = parseJson(out) + log.info("http_stream 2pc result: ${out}".toString()) + assertEquals(code, 0) + assertEquals("success", json2pc.status.toLowerCase()) + } + + def wait_for_publish = {txnId, waitSecond -> + String st = "PREPARE" + while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && waitSecond > 0) { + Thread.sleep(1000) + waitSecond -= 1 + def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" + assertNotNull(result) + st = result[0].TransactionStatus + } + log.info("Stream load with txn ${txnId} is ${st}") + assertEquals(st, "VISIBLE") + } + def txnId1, txnId2, txnId3 + + String load1 = """1,100,1 +2,200,1 +3,50,1 +""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,v1,__DORIS_DELETE_SIGN__' + set 'strict_mode', "false" + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FIXED_COLUMNS' + inputStream new ByteArrayInputStream(load1.getBytes()) + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId1 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + String load2 = """4,1""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,__DORIS_DELETE_SIGN__' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FIXED_COLUMNS' + inputStream new ByteArrayInputStream(load2.getBytes()) + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId2 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + + String load3 = """1,987,77777 +2,987,77777 +3,987,77777 +4,987,77777 +""" + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'k,v2,v3' + set 'two_phase_commit', 'true' + set 'unique_key_update_mode', 'UPDATE_FIXED_COLUMNS' + inputStream new ByteArrayInputStream(load3.getBytes()) + time 40000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + + def json = parseJson(result) + txnId3 = json.TxnId + assertEquals("success", json.Status.toLowerCase()) + } + } + + do_streamload_2pc_commit(txnId1) + wait_for_publish(txnId1, 60) + do_streamload_2pc_commit(txnId2) + wait_for_publish(txnId2, 60) + do_streamload_2pc_commit(txnId3) + wait_for_publish(txnId2, 60) + + sql "sync;" + qt_sql "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__ from ${tableName} order by k;" + inspectRows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k,__DORIS_VERSION_COL__,__DORIS_SEQUENCE_COL__;" +} diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.groovy new file mode 100644 index 00000000000000..05b241c5ceca2c --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_read_from_old.groovy @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_partial_update_seq_read_from_old') { + def inspect_rows = { sqlStr -> + sql "set skip_delete_sign=true;" + sql "set skip_delete_bitmap=true;" + sql "sync" + qt_inspect sqlStr + sql "set skip_delete_sign=false;" + sql "set skip_delete_bitmap=false;" + sql "sync" + } + + for (def use_row_store : [false, true]) { + logger.info("current params: use_row_store: ${use_row_store}") + def tableName = "test_partial_update_seq_read_from_old_${use_row_store}" + sql """ DROP TABLE IF EXISTS ${tableName} force;""" + sql """ CREATE TABLE ${tableName} ( + `k` int(11) NULL, + `v1` BIGINT NULL, + `v2` BIGINT NULL, + `v3` BIGINT NOT NULL DEFAULT "9876", + `v4` BIGINT NOT NULL DEFAULT "1234", + `v5` BIGINT NULL + ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "enable_unique_key_skip_bitmap_column" = "true", + "function_column.sequence_col" = "v1", + "store_row_column" = "${use_row_store}"); """ + + sql "insert into ${tableName} values(1,1000,1,1,1,1);" + qt_sql "select k,v1,v2,v3,v4,v5 from ${tableName} order by k;" + + sql "delete from ${tableName} where k=1;" + qt_sql1 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__;" + + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + + sql "insert into ${tableName}(k,v4,v5) values(1,777,8888);" + qt_sql2 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__;" + + sql "insert into ${tableName}(k,v1,v2,v4) values(1,999,-1,-2);" + qt_sql3 "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__ from ${tableName} order by k;" + inspect_rows "select k,v1,v2,v3,v4,v5,__DORIS_SEQUENCE_COL__,__DORIS_DELETE_SIGN__,__DORIS_VERSION_COL__ from ${tableName} order by k,__DORIS_VERSION_COL__;" + } +} \ No newline at end of file