From 84e73d04fe35a960c93a7221f96d126dfd81ab41 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 21 Sep 2023 16:31:36 +0800 Subject: [PATCH] Revert "[Feature](merge-on-write)Support ignore mode for merge-on-write unique table (#21773)" This reverts commit 3ee89aea35726197cb7e94bb4f2c36bc9d50da84. --- be/src/exec/tablet_info.cpp | 8 +- be/src/exec/tablet_info.h | 2 - be/src/http/action/stream_load.cpp | 7 - be/src/http/http_common.h | 1 - be/src/olap/rowset/beta_rowset_writer.cpp | 2 +- be/src/olap/rowset_builder.cpp | 1 - be/src/olap/tablet.cpp | 74 +++++----- be/src/olap/tablet_schema.cpp | 2 - be/src/olap/tablet_schema.h | 9 +- .../Load/STREAM-LOAD.md | 2 - .../Manipulation/INSERT.md | 4 +- .../Load/STREAM-LOAD.md | 2 - .../Manipulation/INSERT.md | 5 +- fe/fe-core/src/main/cup/sql_parser.cup | 21 +-- .../org/apache/doris/analysis/DeleteStmt.java | 3 +- .../doris/analysis/NativeInsertStmt.java | 26 +--- .../org/apache/doris/analysis/UpdateStmt.java | 3 +- .../doris/load/loadv2/LoadingTaskPlanner.java | 2 +- .../commands/InsertIntoTableCommand.java | 3 +- .../apache/doris/planner/OlapTableSink.java | 18 +-- .../doris/planner/StreamLoadPlanner.java | 8 +- .../org/apache/doris/task/LoadTaskInfo.java | 4 - .../org/apache/doris/task/StreamLoadTask.java | 8 -- .../doris/planner/OlapTableSinkTest.java | 8 +- gensrc/proto/descriptors.proto | 1 - gensrc/proto/olap_file.proto | 1 - gensrc/thrift/Descriptors.thrift | 1 - gensrc/thrift/FrontendService.thrift | 1 - .../data/insert_p0/insert_ignore.out | 36 ----- .../data/unique_with_mow_p0/ignore_mode.csv | 10 -- .../data/unique_with_mow_p0/ignore_mode2.csv | 2 - .../unique_with_mow_p0/test_ignore_mode.out | 20 --- .../suites/insert_p0/insert_ignore.groovy | 132 ------------------ .../test_ignore_mode.groovy | 112 --------------- 34 files changed, 62 insertions(+), 477 deletions(-) delete mode 100644 regression-test/data/insert_p0/insert_ignore.out delete mode 100644 regression-test/data/unique_with_mow_p0/ignore_mode.csv delete mode 100644 regression-test/data/unique_with_mow_p0/ignore_mode2.csv delete mode 100644 regression-test/data/unique_with_mow_p0/test_ignore_mode.out delete mode 100644 regression-test/suites/insert_p0/insert_ignore.groovy delete mode 100644 regression-test/suites/unique_with_mow_p0/test_ignore_mode.groovy diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index 64c80c41af064a..71ca504d3a010a 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -124,7 +124,6 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { _version = pschema.version(); _is_partial_update = pschema.partial_update(); _is_strict_mode = pschema.is_strict_mode(); - _is_unique_key_ignore_mode = pschema.is_unique_key_ignore_mode(); for (auto& col : pschema.partial_update_input_columns()) { _partial_update_input_columns.insert(col); @@ -177,9 +176,9 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { _table_id = tschema.table_id; _version = tschema.version; _is_partial_update = tschema.is_partial_update; - _is_strict_mode = tschema.__isset.is_strict_mode && tschema.is_strict_mode; - _is_unique_key_ignore_mode = - tschema.__isset.is_unique_key_ignore_mode && tschema.is_unique_key_ignore_mode; + if (tschema.__isset.is_strict_mode) { + _is_strict_mode = tschema.is_strict_mode; + } for (auto& tcolumn : tschema.partial_update_input_columns) { _partial_update_input_columns.insert(tcolumn); @@ -247,7 +246,6 @@ void OlapTableSchemaParam::to_protobuf(POlapTableSchemaParam* pschema) const { pschema->set_version(_version); pschema->set_partial_update(_is_partial_update); pschema->set_is_strict_mode(_is_strict_mode); - pschema->set_is_unique_key_ignore_mode(_is_unique_key_ignore_mode); for (auto col : _partial_update_input_columns) { *pschema->add_partial_update_input_columns() = col; } diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index 42e637728484e0..3e6ab7b94be922 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -90,7 +90,6 @@ class OlapTableSchemaParam { return _partial_update_input_columns; } bool is_strict_mode() const { return _is_strict_mode; } - bool is_unique_key_ignore_mode() const { return _is_unique_key_ignore_mode; } std::string debug_string() const; private: @@ -105,7 +104,6 @@ class OlapTableSchemaParam { bool _is_partial_update = false; std::set _partial_update_input_columns; bool _is_strict_mode = false; - bool _is_unique_key_ignore_mode = false; }; using OlapTableIndexTablets = TOlapTableIndexTablets; diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp index e1f721ffc3c15b..b4d53b74c437f0 100644 --- a/be/src/http/action/stream_load.cpp +++ b/be/src/http/action/stream_load.cpp @@ -555,13 +555,6 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req, bool value = iequal(http_req->header(HTTP_MEMTABLE_ON_SINKNODE), "true"); request.__set_memtable_on_sink_node(value); } - if (!http_req->header(HTTP_IGNORE_MODE).empty()) { - if (iequal(http_req->header(HTTP_IGNORE_MODE), "true")) { - request.__set_ignore_mode(true); - } else { - request.__set_ignore_mode(false); - } - } #ifndef BE_TEST // plan this load diff --git a/be/src/http/http_common.h b/be/src/http/http_common.h index 6ccaf60736f55b..bcbfa33e1011db 100644 --- a/be/src/http/http_common.h +++ b/be/src/http/http_common.h @@ -58,7 +58,6 @@ static const std::string HTTP_SKIP_LINES = "skip_lines"; static const std::string HTTP_COMMENT = "comment"; static const std::string HTTP_ENABLE_PROFILE = "enable_profile"; static const std::string HTTP_PARTIAL_COLUMNS = "partial_columns"; -static const std::string HTTP_IGNORE_MODE = "ignore_mode"; static const std::string HTTP_SQL = "sql"; static const std::string HTTP_TWO_PHASE_COMMIT = "two_phase_commit"; static const std::string HTTP_TXN_ID_KEY = "txn_id"; diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 82514809b4e246..2019ea7c349b17 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -149,7 +149,7 @@ Status BetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { OlapStopWatch watch; RETURN_IF_ERROR(_context.tablet->calc_delete_bitmap( rowset, segments, specified_rowsets, _context.mow_context->delete_bitmap, - _context.mow_context->max_version, nullptr, nullptr)); + _context.mow_context->max_version, nullptr)); size_t total_rows = std::accumulate( segments.begin(), segments.end(), 0, [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); }); diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 2787acc30ff474..1de133d3965d45 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -325,7 +325,6 @@ void RowsetBuilder::_build_current_tablet_schema(int64_t index_id, _tablet_schema->set_partial_update_info(table_schema_param->is_partial_update(), table_schema_param->partial_update_input_columns()); _tablet_schema->set_is_strict_mode(table_schema_param->is_strict_mode()); - _tablet_schema->set_is_unique_key_ignore_mode(table_schema_param->is_unique_key_ignore_mode()); } } // namespace doris diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 2946d6e84ac4af..d3d82648017f1f 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -107,7 +107,6 @@ #include "olap/txn_manager.h" #include "olap/types.h" #include "olap/utils.h" -#include "runtime/define_primitive_type.h" #include "segment_loader.h" #include "service/point_query_executor.h" #include "util/bvar_helper.h" @@ -2887,7 +2886,6 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, Version dummy_version(end_version + 1, end_version + 1); auto rowset_schema = rowset->tablet_schema(); bool is_partial_update = rowset_schema->is_partial_update(); - bool is_unique_key_ignore_mode = rowset_schema->is_unique_key_ignore_mode(); // use for partial update PartialUpdateReadPlan read_plan_ori; PartialUpdateReadPlan read_plan_update; @@ -2955,50 +2953,42 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, if (st.is()) { continue; } - if (UNLIKELY(is_unique_key_ignore_mode)) { - if (st.is() || st.is()) { - delete_bitmap->add({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, - row_id); - } - } else { - // sequence id smaller than the previous one, so delete current row - if (st.is()) { - delete_bitmap->add({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, - row_id); - continue; - } else if (is_partial_update && rowset_writer != nullptr) { - // In publish version, record rows to be deleted for concurrent update - // For example, if version 5 and 6 update a row, but version 6 only see - // version 4 when write, and when publish version, version 5's value will - // be marked as deleted and it's update is losed. - // So here we should read version 5's columns and build a new row, which is - // consists of version 6's update columns and version 5's origin columns - // here we build 2 read plan for ori values and update values - prepare_to_read(loc, pos, &read_plan_ori); - prepare_to_read(RowLocation {rowset_id, seg->id(), row_id}, pos, - &read_plan_update); - rsid_to_rowset[rowset_find->rowset_id()] = rowset_find; - ++pos; - // delete bitmap will be calculate when memtable flush and - // publish. The two stages may see different versions. - // When there is sequence column, the currently imported data - // of rowset may be marked for deletion at memtablet flush or - // publish because the seq column is smaller than the previous - // rowset. - // just set 0 as a unified temporary version number, and update to - // the real version number later. - delete_bitmap->add( - {loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, - loc.row_id); - delete_bitmap->add({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, - row_id); - continue; - } - // when st = ok + + // sequence id smaller than the previous one, so delete current row + if (st.is()) { + delete_bitmap->add({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, + row_id); + continue; + } else if (is_partial_update && rowset_writer != nullptr) { + // In publish version, record rows to be deleted for concurrent update + // For example, if version 5 and 6 update a row, but version 6 only see + // version 4 when write, and when publish version, version 5's value will + // be marked as deleted and it's update is losed. + // So here we should read version 5's columns and build a new row, which is + // consists of version 6's update columns and version 5's origin columns + // here we build 2 read plan for ori values and update values + prepare_to_read(loc, pos, &read_plan_ori); + prepare_to_read(RowLocation {rowset_id, seg->id(), row_id}, pos, &read_plan_update); + rsid_to_rowset[rowset_find->rowset_id()] = rowset_find; + ++pos; + // delete bitmap will be calculate when memtable flush and + // publish. The two stages may see different versions. + // When there is sequence column, the currently imported data + // of rowset may be marked for deletion at memtablet flush or + // publish because the seq column is smaller than the previous + // rowset. + // just set 0 as a unified temporary version number, and update to + // the real version number later. delete_bitmap->add( {loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, loc.row_id); + delete_bitmap->add({rowset_id, seg->id(), DeleteBitmap::TEMP_VERSION_COMMON}, + row_id); + continue; } + // when st = ok + delete_bitmap->add({loc.rowset_id, loc.segment_id, DeleteBitmap::TEMP_VERSION_COMMON}, + loc.row_id); } remaining -= num_read; } diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index a87b86ced712d8..6f25b20e650eff 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -759,7 +759,6 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema) { _compression_type = schema.compression_type(); _schema_version = schema.schema_version(); _is_partial_update = schema.is_partial_update(); - _is_unique_key_ignore_mode = schema.is_unique_key_ignore_mode(); for (auto& col_name : schema.partial_update_input_columns()) { _partial_update_input_columns.emplace(col_name); } @@ -918,7 +917,6 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_compression_type(_compression_type); tablet_schema_pb->set_version_col_idx(_version_col_idx); tablet_schema_pb->set_is_partial_update(_is_partial_update); - tablet_schema_pb->set_is_unique_key_ignore_mode(_is_unique_key_ignore_mode); for (auto& col : _partial_update_input_columns) { *tablet_schema_pb->add_partial_update_input_columns() = col; } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 57faee4d827584..72d0636f6e472a 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -363,12 +363,8 @@ class TabletSchema { } void set_is_strict_mode(bool is_strict_mode) { _is_strict_mode = is_strict_mode; } bool is_strict_mode() const { return _is_strict_mode; } - void set_is_unique_key_ignore_mode(bool is_unique_key_ignore_mode) { - _is_unique_key_ignore_mode = is_unique_key_ignore_mode; - } - bool is_unique_key_ignore_mode() const { return _is_unique_key_ignore_mode; } - std::vector get_missing_cids() { return _missing_cids; } - std::vector get_update_cids() { return _update_cids; } + std::vector get_missing_cids() const { return _missing_cids; } + std::vector get_update_cids() const { return _update_cids; } private: friend bool operator==(const TabletSchema& a, const TabletSchema& b); @@ -415,7 +411,6 @@ class TabletSchema { // to generate a new row, only available in non-strict mode bool _can_insert_new_rows_in_partial_update = true; bool _is_strict_mode = false; - bool _is_unique_key_ignore_mode = false; }; bool operator==(const TabletSchema& a, const TabletSchema& b); diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md index cdc44ae08687ee..039462c219b536 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md @@ -156,8 +156,6 @@ separated by commas. 28. comment: String type, the default value is "". -29. ignore_mode: Ignore mode, only effective when the target table is a unique table with merge-on-write enabled. When insert ignore mode is enabled, for the inserted rows, if the key of the row does not exist in the table, the row will be inserted. If the key already exists in the table, the row will be discarded. When sequence columns exists in the target table, the ignore mode can't be enabled in stream load. - ### Example 1. Import the data in the local file 'testData' into the table 'testTbl' in the database 'testDb', and use Label for deduplication. Specify a timeout of 100 seconds diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md index dd11abb99cd5e4..441a393816063b 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md @@ -35,7 +35,7 @@ INSERT The change statement is to complete the data insertion operation. ```sql -INSERT [IGNORE] INTO table_name +INSERT INTO table_name [ PARTITION (p1, ...) ] [ WITH LABEL label] [ (column [, ...]) ] @@ -44,7 +44,7 @@ INSERT [IGNORE] INTO table_name ```` Parameters -> IGNORE: insert ignore mode, only effective when the target table is a unique table with merge-on-write enabled. When insert ignore mode is enabled, for the inserted rows, if the key of the row does not exist in the table, the row will be inserted. If the key already exists in the table, the row will be discarded. When sequence column exists in the target table, the `insert ignore` statements are forbidden. + > tablet_name: The destination table for importing data. Can be of the form `db_name.table_name` > > partitions: Specify the partitions to be imported, which must be partitions that exist in `table_name`. Multiple partition names are separated by commas diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md index 00c622cdd5fcc7..8ca18f74df6b18 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md @@ -152,8 +152,6 @@ curl --location-trusted -u user:passwd [-H ""...] -T data.file -XPUT http://fe_h 28. comment: 字符串类型, 默认值为空. 给任务增加额外的信息. -29. ignore_mode: ignore模式，仅当目标表为开启merge-on-write的unique表时有效。开启后，对于插入的行，如果该行的key在表中不存在，则插入该行数据。如果key在表中不存在，则丢弃这行数据。当目标表中存在sequence列时stream无法开启ignore mode。 - ### Example 1. 将本地文件'testData'中的数据导入到数据库'testDb'中'testTbl'的表，使用Label用于去重。指定超时时间为 100 秒 diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md index 0f914407f6b979..78ae40a1efb64e 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md @@ -35,7 +35,7 @@ INSERT 该语句是完成数据插入操作。 ```sql -INSERT [IGNORE] INTO table_name +INSERT INTO table_name [ PARTITION (p1, ...) ] [ WITH LABEL label] [ (column [, ...]) ] @@ -44,8 +44,7 @@ INSERT [IGNORE] INTO table_name ``` Parameters -> IGNORE: insert ignore模式，仅当目标表为开启merge-on-write的unique表时有效。开启后，对于插入的行，如果该行的key在表中不存在，则插入该行数据。如果key在表中不存在，则丢弃这行数据。当目标表中存在sequence列时无法通过insert ignore语句进行插入操作。 -> + > tablet_name: 导入数据的目的表。可以是 `db_name.table_name` 形式 > > partitions: 指定待导入的分区，必须是 `table_name` 中存在的分区，多个分区名称用逗号分隔 diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 437b710e7ffdcc..bd7800196d8e8e 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -889,7 +889,7 @@ nonterminal ParseNode load_property; nonterminal List opt_load_property_list; // Boolean -nonterminal Boolean opt_negative, opt_is_allow_null, opt_is_key, opt_read_only, opt_aggregate, opt_local, opt_is_auto_inc, opt_is_insert_ignore; +nonterminal Boolean opt_negative, opt_is_allow_null, opt_is_key, opt_read_only, opt_aggregate, opt_local, opt_is_auto_inc; nonterminal String opt_from_rollup, opt_to_rollup; nonterminal ColumnPosition opt_col_pos; @@ -3727,17 +3727,6 @@ opt_is_auto_inc ::= RESULT = true; :} ; - -opt_is_insert_ignore ::= - {: - RESULT = false; - :} - | KW_IGNORE - {: - RESULT = true; - :} - ; - opt_comment ::= /* empty */ {: @@ -4825,16 +4814,16 @@ insert_overwrite_stmt ::= // Insert statement insert_stmt ::= - KW_INSERT opt_is_insert_ignore:is_insert_ignore KW_INTO insert_target:target opt_with_label:label opt_col_list:cols opt_plan_hints:hints insert_source:source + KW_INSERT KW_INTO insert_target:target opt_with_label:label opt_col_list:cols opt_plan_hints:hints insert_source:source {: - RESULT = new NativeInsertStmt(target, label, cols, source, hints, is_insert_ignore); + RESULT = new NativeInsertStmt(target, label, cols, source, hints); :} // TODO(zc) add default value for SQL-2003 // | KW_INSERT KW_INTO insert_target:target KW_DEFAULT KW_VALUES | /* used for group commit */ - KW_INSERT opt_is_insert_ignore:is_insert_ignore KW_INTO INTEGER_LITERAL:table_id opt_with_label:label opt_col_list:cols opt_plan_hints:hints insert_source:source + KW_INSERT KW_INTO INTEGER_LITERAL:table_id opt_with_label:label opt_col_list:cols opt_plan_hints:hints insert_source:source {: - RESULT = new NativeInsertStmt(table_id, label, cols, source, hints, is_insert_ignore); + RESULT = new NativeInsertStmt(table_id, label, cols, source, hints); :} ; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java index 2a282be2778eef..768b3aecbe192a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java @@ -201,8 +201,7 @@ private void constructInsertStmt() throws AnalysisException { cols, new InsertSource(selectStmt), null, - isPartialUpdate, - false); + isPartialUpdate); ((NativeInsertStmt) insertStmt).setIsFromDeleteOrUpdateStmt(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java index fb5c9a14c1bde1..ad1a2f1e527384 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java @@ -166,7 +166,6 @@ public class NativeInsertStmt extends InsertStmt { // true if be generates an insert from group commit tvf stmt and executes to load data public boolean isInnerGroupCommit = false; - private boolean isInsertIgnore = false; private boolean isFromDeleteOrUpdateStmt = false; public NativeInsertStmt(InsertTarget target, String label, List cols, InsertSource source, @@ -188,26 +187,6 @@ public NativeInsertStmt(long tableId, String label, List cols, InsertSou this.tableId = tableId; } - public NativeInsertStmt(long tableId, String label, List cols, InsertSource source, - List hints, boolean isInsertIgnore) { - this(new InsertTarget(new TableName(null, null, null), null), label, cols, source, hints, isInsertIgnore); - this.tableId = tableId; - } - - public NativeInsertStmt(InsertTarget target, String label, List cols, InsertSource source, - List hints, boolean isInsertIgnore) { - super(new LabelName(null, label), null, null); - this.tblName = target.getTblName(); - this.targetPartitionNames = target.getPartitionNames(); - this.label = new LabelName(null, label); - this.queryStmt = source.getQueryStmt(); - this.planHints = hints; - this.isInsertIgnore = isInsertIgnore; - this.targetColumnNames = cols; - this.isValuesOrConstantSelect = (queryStmt instanceof SelectStmt - && ((SelectStmt) queryStmt).getTableRefs().isEmpty()); - } - // Ctor for CreateTableAsSelectStmt and InsertOverwriteTableStmt public NativeInsertStmt(TableName name, PartitionNames targetPartitionNames, LabelName label, QueryStmt queryStmt, List planHints, List targetColumnNames) { @@ -222,11 +201,10 @@ public NativeInsertStmt(TableName name, PartitionNames targetPartitionNames, Lab } public NativeInsertStmt(InsertTarget target, String label, List cols, InsertSource source, - List hints, boolean isPartialUpdate, boolean isInsertIgnore) { + List hints, boolean isPartialUpdate) { this(target, label, cols, source, hints); this.isPartialUpdate = isPartialUpdate; this.partialUpdateCols.addAll(cols); - this.isInsertIgnore = isInsertIgnore; } public boolean isValuesOrConstantSelect() { @@ -409,7 +387,7 @@ public void analyze(Analyzer analyzer) throws UserException { boolean isInsertStrict = analyzer.getContext().getSessionVariable().getEnableInsertStrict() && !isFromDeleteOrUpdateStmt; sink.init(loadId, transactionId, db.getId(), timeoutSecond, - sendBatchParallelism, false, isInsertStrict, isInsertIgnore); + sendBatchParallelism, false, isInsertStrict); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java index 0decc606ef2a67..021840f8162958 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UpdateStmt.java @@ -125,8 +125,7 @@ private void constructInsertStmt() { cols, new InsertSource(selectStmt), null, - isPartialUpdate, - false); + isPartialUpdate); ((NativeInsertStmt) insertStmt).setIsFromDeleteOrUpdateStmt(true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index 7cbeb589195961..d14b5d6d7809f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -202,7 +202,7 @@ public void plan(TUniqueId loadId, List> fileStatusesLis List partitionIds = getAllPartitionIds(); OlapTableSink olapTableSink = new OlapTableSink(table, destTupleDesc, partitionIds, Config.enable_single_replica_load); - olapTableSink.init(loadId, txnId, dbId, timeoutS, sendBatchParallelism, false, strictMode, false); + olapTableSink.init(loadId, txnId, dbId, timeoutS, sendBatchParallelism, false, strictMode); olapTableSink.setPartialUpdateInputColumns(isPartialUpdate, partialUpdateInputColumns); olapTableSink.complete(analyzer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java index e6affcb545ffda..f87b79308c32c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java @@ -154,8 +154,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { ctx.getExecTimeout(), ctx.getSessionVariable().getSendBatchParallelism(), false, - isStrictMode, - false); + isStrictMode); sink.complete(new Analyzer(Env.getCurrentEnv(), ctx)); TransactionState state = Env.getCurrentGlobalTransactionMgr().getTransactionState( diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 4b3ff601582a40..4ef1aa9015aef4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -26,7 +26,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.HashDistributionInfo; import org.apache.doris.catalog.Index; -import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; @@ -106,8 +105,6 @@ public class OlapTableSink extends DataSink { private boolean isStrictMode = false; - private boolean isUniqueKeyIgnoreMode = false; - public OlapTableSink(OlapTable dstTable, TupleDescriptor tupleDescriptor, List partitionIds, boolean singleReplicaLoad) { this.dstTable = dstTable; @@ -117,7 +114,7 @@ public OlapTableSink(OlapTable dstTable, TupleDescriptor tupleDescriptor, List partitionIds = getAllPartitionIds(); OlapTableSink olapTableSink = new OlapTableSink(destTable, tupleDesc, partitionIds, Config.enable_single_replica_load); - olapTableSink.init(loadId, taskInfo.getTxnId(), db.getId(), timeout, taskInfo.getSendBatchParallelism(), - taskInfo.isLoadToSingleTablet(), taskInfo.isStrictMode(), taskInfo.isIgnoreMode()); + olapTableSink.init(loadId, taskInfo.getTxnId(), db.getId(), timeout, + taskInfo.getSendBatchParallelism(), taskInfo.isLoadToSingleTablet(), taskInfo.isStrictMode()); olapTableSink.setPartialUpdateInputColumns(isPartialUpdate, partialUpdateInputColumns); olapTableSink.complete(analyzer); @@ -465,8 +465,8 @@ public TPipelineFragmentParams planForPipeline(TUniqueId loadId, int fragmentIns List partitionIds = getAllPartitionIds(); OlapTableSink olapTableSink = new OlapTableSink(destTable, tupleDesc, partitionIds, Config.enable_single_replica_load); - olapTableSink.init(loadId, taskInfo.getTxnId(), db.getId(), timeout, taskInfo.getSendBatchParallelism(), - taskInfo.isLoadToSingleTablet(), taskInfo.isStrictMode(), taskInfo.isIgnoreMode()); + olapTableSink.init(loadId, taskInfo.getTxnId(), db.getId(), timeout, + taskInfo.getSendBatchParallelism(), taskInfo.isLoadToSingleTablet(), taskInfo.isStrictMode()); olapTableSink.setPartialUpdateInputColumns(isPartialUpdate, partialUpdateInputColumns); olapTableSink.complete(analyzer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java index 9f21de25b014bd..3174e4d5c6b6a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/LoadTaskInfo.java @@ -125,10 +125,6 @@ default boolean isMemtableOnSinkNode() { return false; } - default boolean isIgnoreMode() { - return false; - } - class ImportColumnDescs { public List descs = Lists.newArrayList(); public boolean isColumnDescsRewrited = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java index 4c95da21023760..c99c720ee0d4c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/StreamLoadTask.java @@ -84,7 +84,6 @@ public class StreamLoadTask implements LoadTaskInfo { private List hiddenColumns; private boolean trimDoubleQuotes = false; private boolean isPartialUpdate = false; - private boolean isIgnoreMode = false; private int skipLines = 0; private boolean enableProfile = false; @@ -309,10 +308,6 @@ public void setMemtableOnSinkNode(boolean memtableOnSinkNode) { this.memtableOnSinkNode = memtableOnSinkNode; } - public boolean isIgnoreMode() { - return isIgnoreMode; - } - public static StreamLoadTask fromTStreamLoadPutRequest(TStreamLoadPutRequest request) throws UserException { StreamLoadTask streamLoadTask = new StreamLoadTask(request.getLoadId(), request.getTxnId(), request.getFileType(), request.getFormatType(), @@ -449,9 +444,6 @@ private void setOptionalFromTSLPutRequest(TStreamLoadPutRequest request) throws if (request.isSetMemtableOnSinkNode()) { this.memtableOnSinkNode = request.isMemtableOnSinkNode(); } - if (request.isSetIgnoreMode()) { - isIgnoreMode = request.isIgnoreMode(); - } } // used for stream load diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java index 44a0f0292877cf..dc98026a00eb2c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapTableSinkTest.java @@ -107,7 +107,7 @@ public void testSinglePartition() throws UserException { new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM)); dstTable.getPartitionInfo().setIsMutable(partition.getId(), true); OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(2L), false); - sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false, false); + sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false); sink.complete(null); LOG.info("sink is {}", sink.toThrift()); LOG.info("{}", sink.getExplainString("", TExplainLevel.NORMAL)); @@ -144,7 +144,7 @@ public void testRangePartition( }; OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(p1.getId()), false); - sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false, false); + sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false); try { sink.complete(null); } catch (UserException e) { @@ -169,7 +169,7 @@ public void testRangeUnknownPartition( }; OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(unknownPartId), false); - sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false, false); + sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false); sink.complete(null); LOG.info("sink is {}", sink.toThrift()); LOG.info("{}", sink.getExplainString("", TExplainLevel.NORMAL)); @@ -206,7 +206,7 @@ public void testListPartition( }; OlapTableSink sink = new OlapTableSink(dstTable, tuple, Lists.newArrayList(p1.getId()), false); - sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false, false); + sink.init(new TUniqueId(1, 2), 3, 4, 1000, 1, false, false); try { sink.complete(null); } catch (UserException e) { diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index b2e76fc0194f2a..abebf8fde5e6a1 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -67,6 +67,5 @@ message POlapTableSchemaParam { optional bool partial_update = 7; repeated string partial_update_input_columns = 8; optional bool is_strict_mode = 9 [default = false]; - optional bool is_unique_key_ignore_mode = 10 [default = false]; }; diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index bc968e87ee12ee..fe5f76f6b4f18c 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -254,7 +254,6 @@ message TabletSchemaPB { repeated string partial_update_input_columns = 21; optional bool enable_single_replica_compaction = 22 [default=false]; optional bool skip_write_index_on_load = 23 [default=false]; - optional bool is_unique_key_ignore_mode = 24 [default=false]; } enum TabletStatePB { diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 10521af03318c2..fa391febdaef22 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -230,7 +230,6 @@ struct TOlapTableSchemaParam { 8: optional bool is_partial_update 9: optional list partial_update_input_columns 10: optional bool is_strict_mode = false; - 11: optional bool is_unique_key_ignore_mode = false; } struct TTabletLocation { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 29739450a146a1..2804555caa9c11 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -639,7 +639,6 @@ struct TStreamLoadPutRequest { // only valid when file type is CSV 52: optional i8 escape 53: optional bool memtable_on_sink_node; - 54: optional bool ignore_mode = false } struct TStreamLoadPutResult { diff --git a/regression-test/data/insert_p0/insert_ignore.out b/regression-test/data/insert_p0/insert_ignore.out deleted file mode 100644 index bb38f60c9cc4be..00000000000000 --- a/regression-test/data/insert_p0/insert_ignore.out +++ /dev/null @@ -1,36 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !origin_data -- -1 kevin 18 shenzhen 400 -2 bob 20 beijing 500 -3 alice 22 shanghai 600 -4 jack 24 hangzhou 700 -5 tom 26 guanzhou 800 - --- !after_insert_ignore -- -1 kevin 18 shenzhen 400 -2 bob 20 beijing 500 -3 alice 22 shanghai 600 -4 jack 24 hangzhou 700 -5 tom 26 guanzhou 800 -10 alex 28 shenzhen 1111 -20 leo 30 beijing 2222 -30 sam 32 shanghai 3333 -40 Ruth 34 hangzhou 4444 -50 cynthia 36 guanzhou 8000 - --- !origin_data -- -1 1 - --- !delete_a_row -- - --- !after_insert_ignore -- -1 3 - --- !sql -- -1 10 - --- !delete_a_row -- - --- !after_insert_ignore -- -1 1 - diff --git a/regression-test/data/unique_with_mow_p0/ignore_mode.csv b/regression-test/data/unique_with_mow_p0/ignore_mode.csv deleted file mode 100644 index 693c484172459c..00000000000000 --- a/regression-test/data/unique_with_mow_p0/ignore_mode.csv +++ /dev/null @@ -1,10 +0,0 @@ -1,"kevin",18,"shenzhen",4000 -10,"alex",28,"shenzhen",1111 -2,"bob",20,"beijing",5000 -20,"leo",30,"beijing",2222 -30,"sam",32,"shanghai",3333 -3,"alice",22,"shanghai",6000 -4,"jack",24,"hangzhou",7000 -40,"Ruth",34,"hangzhou",4444 -5,"tom",26,"guanzhou",8000 -50,"cynthia",36,"guanzhou",8000 \ No newline at end of file diff --git a/regression-test/data/unique_with_mow_p0/ignore_mode2.csv b/regression-test/data/unique_with_mow_p0/ignore_mode2.csv deleted file mode 100644 index 5711f63304b74a..00000000000000 --- a/regression-test/data/unique_with_mow_p0/ignore_mode2.csv +++ /dev/null @@ -1,2 +0,0 @@ -1,4000 -2,5000 diff --git a/regression-test/data/unique_with_mow_p0/test_ignore_mode.out b/regression-test/data/unique_with_mow_p0/test_ignore_mode.out deleted file mode 100644 index 5c72f099d99f41..00000000000000 --- a/regression-test/data/unique_with_mow_p0/test_ignore_mode.out +++ /dev/null @@ -1,20 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !origin_data -- -1 kevin 18 shenzhen 400 -2 bob 20 beijing 500 -3 alice 22 shanghai 600 -4 jack 24 hangzhou 700 -5 tom 26 guanzhou 800 - --- !after_ignore_mode_stream_load -- -1 kevin 18 shenzhen 400 -2 bob 20 beijing 500 -3 alice 22 shanghai 600 -4 jack 24 hangzhou 700 -5 tom 26 guanzhou 800 -10 "alex" 28 "shenzhen" 1111 -20 "leo" 30 "beijing" 2222 -30 "sam" 32 "shanghai" 3333 -40 "Ruth" 34 "hangzhou" 4444 -50 "cynthia" 36 "guanzhou" 8000 - diff --git a/regression-test/suites/insert_p0/insert_ignore.groovy b/regression-test/suites/insert_p0/insert_ignore.groovy deleted file mode 100644 index 37cd7707f2e4eb..00000000000000 --- a/regression-test/suites/insert_p0/insert_ignore.groovy +++ /dev/null @@ -1,132 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_insert_ignore") { - - def tableName = "test_insert_ignore1" - sql """ DROP TABLE IF EXISTS ${tableName} FORCE;""" - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(10) NULL, - `age` int(11) NULL DEFAULT "20", - `city` varchar(10) NOT NULL DEFAULT "beijing", - `balance` decimalv3(9, 0) NULL - ) ENGINE = OLAP UNIQUE KEY(`id`) - COMMENT 'OLAP' DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "storage_format" = "V2", - "enable_unique_key_merge_on_write" = "true", - "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" - ); - """ - sql """insert into ${tableName} values - (1,"kevin",18,"shenzhen",400), - (2,"bob",20,"beijing",500), - (3,"alice",22,"shanghai",600), - (4,"jack",24,"hangzhou",700), - (5,"tom",26,"guanzhou",800);""" - qt_origin_data "select * from ${tableName} order by id;" - - // some rows are with existing keys, some are not - sql """insert ignore into ${tableName} values - (1,"kevin",18,"shenzhen",4000), - (10,"alex",28,"shenzhen",1111), - (2,"bob",20,"beijing",5000), - (20,"leo",30,"beijing",2222), - (30,"sam",32,"shanghai",3333), - (3,"alice",22,"shanghai",6000), - (4,"jack",24,"hangzhou",7000), - (40,"Ruth",34,"hangzhou",4444), - (5,"tom",26,"guanzhou",8000), - (50,"cynthia",36,"guanzhou",8000);""" - - qt_after_insert_ignore "select * from ${tableName} order by id;" - sql """ DROP TABLE IF EXISTS ${tableName};""" - - def tableName2 = "test_insert_ignore2" - sql """ DROP TABLE IF EXISTS ${tableName2} FORCE; """ - sql """CREATE TABLE IF NOT EXISTS ${tableName2} ( - `uid` BIGINT NULL, - `v1` BIGINT NULL - ) - UNIQUE KEY(uid) - DISTRIBUTED BY HASH(uid) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "replication_num" = "1" - );""" - - sql "insert into ${tableName2} values(1,1);" - qt_origin_data "select * from ${tableName2} order by uid;" - - sql "insert into ${tableName2}(uid, v1, __DORIS_DELETE_SIGN__) values(1, 2, 1);" - qt_delete_a_row "select * from ${tableName2} order by uid;" - - sql "insert ignore into ${tableName2} values(1,3);" - qt_after_insert_ignore "select * from ${tableName2} order by uid;" - - sql "insert into ${tableName2} values(1,10);" - qt_sql "select * from ${tableName2} order by uid;" - - sql "insert into ${tableName2}(uid, v1, __DORIS_DELETE_SIGN__) values(1, 1, 1);" - qt_delete_a_row "select * from ${tableName2} order by uid;" - - sql "insert ignore into ${tableName2} values(1,1);" - qt_after_insert_ignore "select * from ${tableName2} order by uid;" - sql """ DROP TABLE IF EXISTS ${tableName2}; """ - - - // test illigal cases - def tableName3 = "test_insert_ignore3" - sql """ DROP TABLE IF EXISTS ${tableName3} FORCE; """ - sql """CREATE TABLE IF NOT EXISTS ${tableName3} ( - `uid` BIGINT NULL, - `v1` BIGINT NULL - ) UNIQUE KEY(uid) - DISTRIBUTED BY HASH(uid) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "false", - "replication_num" = "1" - );""" - sql "insert into ${tableName3} values(1,1);" - test { - sql "insert ignore into ${tableName3} values(1,3);" - exception "ignore mode can only be enabled if the target table is a unique table with merge-on-write enabled." - } - - def tableName4 = "test_insert_ignore4" - sql """ DROP TABLE IF EXISTS ${tableName4} FORCE; """ - sql """CREATE TABLE IF NOT EXISTS ${tableName4} ( - `uid` BIGINT NULL, - `v1` BIGINT NULL - ) UNIQUE KEY(uid) - DISTRIBUTED BY HASH(uid) BUCKETS 1 - PROPERTIES ( - "enable_unique_key_merge_on_write" = "true", - "replication_num" = "1", - "function_column.sequence_col" = 'v1' - );""" - sql "insert into ${tableName4} values(1,1);" - test { - sql "insert ignore into ${tableName4} values(1,3);" - exception "ignore mode can't be used if the target table has sequence column, but table[${tableName4}] has sequnce column." - } -} diff --git a/regression-test/suites/unique_with_mow_p0/test_ignore_mode.groovy b/regression-test/suites/unique_with_mow_p0/test_ignore_mode.groovy deleted file mode 100644 index 94afd0be6b0715..00000000000000 --- a/regression-test/suites/unique_with_mow_p0/test_ignore_mode.groovy +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_mow_table_ignore_mode") { - - def tableName = "test_mow_table_ignore_mode1" - sql """ DROP TABLE IF EXISTS ${tableName} FORCE;""" - sql """ - CREATE TABLE ${tableName} ( - `id` int(11) NULL, - `name` varchar(10) NULL, - `age` int(11) NULL DEFAULT "20", - `city` varchar(10) NOT NULL DEFAULT "beijing", - `balance` decimalv3(9, 0) NULL - ) ENGINE = OLAP UNIQUE KEY(`id`) - COMMENT 'OLAP' DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "storage_format" = "V2", - "enable_unique_key_merge_on_write" = "true", - "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" - ); - """ - sql """insert into ${tableName} values - (1,"kevin",18,"shenzhen",400), - (2,"bob",20,"beijing",500), - (3,"alice",22,"shanghai",600), - (4,"jack",24,"hangzhou",700), - (5,"tom",26,"guanzhou",800);""" - qt_origin_data "select * from ${tableName} order by id;" - - // some rows are with existing keys, some are not - streamLoad { - table "${tableName}" - - set 'column_separator', ',' - set 'format', 'csv' - set 'columns', 'id,name,age,city,balance' - set 'ignore_mode', 'true' - - file 'ignore_mode.csv' - time 10000 // limit inflight 10s - } - sql "sync" - - qt_after_ignore_mode_stream_load "select * from ${tableName} order by id;" - sql """ DROP TABLE IF EXISTS ${tableName};""" - - - // test illegal case - def tableName2 = "test_mow_table_ignore_mode2" - sql """ DROP TABLE IF EXISTS ${tableName2} FORCE;""" - sql """ - CREATE TABLE ${tableName2} ( - `id` int(11) NULL, - `name` varchar(10) NULL, - `age` int(11) NULL DEFAULT "20", - `city` varchar(10) NOT NULL DEFAULT "beijing", - `balance` decimalv3(9, 0) NULL - ) ENGINE = OLAP UNIQUE KEY(`id`) - COMMENT 'OLAP' DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "storage_format" = "V2", - "enable_unique_key_merge_on_write" = "true", - "light_schema_change" = "true", - "disable_auto_compaction" = "false", - "enable_single_replica_compaction" = "false" - );""" - sql """insert into ${tableName2} values - (1,"kevin",18,"shenzhen",400), - (2,"bob",20,"beijing",500), - (3,"alice",22,"shanghai",600), - (4,"jack",24,"hangzhou",700), - (5,"tom",26,"guanzhou",800);""" - // some rows are with existing keys, some are not - streamLoad { - table "${tableName2}" - - set 'column_separator', ',' - set 'format', 'csv' - set 'columns', 'id,balance' - set 'partial_columns', 'true' - set 'ignore_mode', 'true' - - file 'ignore_mode.csv' - time 10000 // limit inflight 10s - - check {result, exception, startTime, endTime -> - assertTrue(exception == null) - def json = parseJson(result) - assertEquals("Fail", json.Status) - assertTrue(json.Message.contains("ignore mode can't be used in partial update.")) - } - } -}