From fbdf556cc4dd03d5d6a8a29d99e4ce4710ea183b Mon Sep 17 00:00:00 2001 From: jacktengg <18241664+jacktengg@users.noreply.github.com> Date: Wed, 28 Sep 2022 16:47:48 +0800 Subject: [PATCH 1/2] Revert "[bugfix](compaction) bug: compaction read delete data as normal data (#13025)" This reverts commit 98f11243cb4c9e20f35d3285cd12fc9a8a91fea3. --- be/src/olap/reader.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp index 5f7dd85f01ce89..c93cbc39712954 100644 --- a/be/src/olap/reader.cpp +++ b/be/src/olap/reader.cpp @@ -824,8 +824,7 @@ OLAPStatus TabletReader::_init_delete_condition(const ReaderParams& read_params) return OLAP_SUCCESS; } OLAPStatus ret = OLAP_SUCCESS; - // if it's READER_QUERY and skip_delete_predicate variable is set, deleted data will be read as normal - if (!read_params.runtime_state || !read_params.runtime_state->skip_delete_predicate()) { + if (read_params.runtime_state && !read_params.runtime_state->skip_delete_predicate()) { ReadLock rdlock(_tablet->get_header_lock()); ret = _delete_handler.init(_tablet->tablet_schema(), _tablet->delete_predicates(), read_params.version.second, this); From 29b796e8201660a9337422e68250f889bb066922 Mon Sep 17 00:00:00 2001 From: jacktengg <18241664+jacktengg@users.noreply.github.com> Date: Wed, 28 Sep 2022 16:48:31 +0800 Subject: [PATCH 2/2] Revert "[improvement](storage) For debugging problems: add session variable skip_storage_engine_merge to treat agg and unique data model as dup model #12973" This reverts commit 7b0313d72e896c31f294bd507420c9b29c1b18da. --- be/src/exec/olap_scanner.cpp | 7 +----- be/src/olap/reader.cpp | 4 ++-- be/src/runtime/runtime_state.h | 9 ------- docs/en/administrator-guide/variables.md | 8 +------ docs/zh-CN/administrator-guide/variables.md | 5 ---- .../org/apache/doris/common/util/Util.java | 4 +--- .../org/apache/doris/qe/SessionVariable.java | 24 ------------------- gensrc/thrift/PaloInternalService.thrift | 6 ----- 8 files changed, 5 insertions(+), 62 deletions(-) diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index 01f455b46d021d..a5f29970eaf176 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -153,12 +153,7 @@ Status OlapScanner::_init_tablet_reader_params( ->rowset_meta() ->is_segments_overlapping()); - if (_runtime_state->skip_storage_engine_merge()) { - _tablet_reader_params.direct_mode = true; - _aggregation = true; - } else { - _tablet_reader_params.direct_mode = single_version || _aggregation; - } + _tablet_reader_params.direct_mode = single_version || _aggregation; RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode)); diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp index c93cbc39712954..9b0b7ceec0db54 100644 --- a/be/src/olap/reader.cpp +++ b/be/src/olap/reader.cpp @@ -823,8 +823,8 @@ OLAPStatus TabletReader::_init_delete_condition(const ReaderParams& read_params) if (read_params.reader_type == READER_CUMULATIVE_COMPACTION) { return OLAP_SUCCESS; } - OLAPStatus ret = OLAP_SUCCESS; - if (read_params.runtime_state && !read_params.runtime_state->skip_delete_predicate()) { + OLAPStatus ret; + { ReadLock rdlock(_tablet->get_header_lock()); ret = _delete_handler.init(_tablet->tablet_schema(), _tablet->delete_predicates(), read_params.version.second, this); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index f96c799898b9c9..b9edc56ea0094a 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -349,15 +349,6 @@ class RuntimeState { return _query_options.enable_enable_exchange_node_parallel_merge; } - bool skip_storage_engine_merge() const { - return _query_options.__isset.skip_storage_engine_merge && - _query_options.skip_storage_engine_merge; - } - - bool skip_delete_predicate() const { - return _query_options.__isset.skip_delete_predicate && _query_options.skip_delete_predicate; - } - const std::vector& tablet_commit_infos() const { return _tablet_commit_infos; } diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md index fb106efa2c2667..3cd6b1a718607c 100644 --- a/docs/en/administrator-guide/variables.md +++ b/docs/en/administrator-guide/variables.md @@ -496,10 +496,4 @@ Translated with www.DeepL.com/Translator (free version) SM4_128_CFB128, SM4_128_OFB, SM4_128_CTR, -``` - -* `skip_storage_engine_merge` - For debugging purpose. In vectorized execution engine, in case of problems of reading data of Aggregate Key model and Unique Key model, setting value to `true` will read data as Duplicate Key model. - -* `skip_delete_predicate` - For debugging purpose. In vectorized execution engine, in case of problems of reading data, setting value to `true` will also read deleted data. +``` \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md index 9c56491b15356b..cef03104569e69 100644 --- a/docs/zh-CN/administrator-guide/variables.md +++ b/docs/zh-CN/administrator-guide/variables.md @@ -490,8 +490,3 @@ SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3); 用于控制是否进行谓词推导。取值有两种：true 和 false。默认情况下关闭，系统不在进行谓词推导，采用原始的谓词进行相关操作。设置为 true 后，进行谓词扩展。 -* `skip_storage_engine_merge` - 用于调试目的。在向量化执行引擎中，当发现读取Aggregate Key模型或者Unique Key模型的数据结果有问题的时候，把此变量的值设置为`true`，将会把Aggregate Key模型或者Unique Key模型的数据当成Duplicate Key模型读取。 - -* `skip_delete_predicate` - 用于调试目的。在向量化执行引擎中，当发现读取表的数据结果有误的时候，把此变量的值设置为`true`，将会把被删除的数据当成正常数据读取。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index c5de3e912347ba..1c5daa9770d245 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -446,9 +446,7 @@ public static InputStream getInputStreamFromUrl(String urlStr, String encodedAut } public static boolean showHiddenColumns() { - return ConnectContext.get() != null && ( - ConnectContext.get().getSessionVariable().showHiddenColumns() - || ConnectContext.get().getSessionVariable().skipStorageEngineMerge()); + return ConnectContext.get() != null && ConnectContext.get().getSessionVariable().showHiddenColumns(); } public static String escapeSingleRegex(String s) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 089047730f310e..6af37a639bd33d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -191,10 +191,6 @@ public class SessionVariable implements Serializable, Writable { public static final String NUM_FREE_BLOCK_IN_SCAN = "num_free_block_in_scan"; - public static final String SKIP_STORAGE_ENGINE_MERGE = "skip_storage_engine_merge"; - - public static final String SKIP_DELETE_PREDICATE = "skip_delete_predicate"; - // session origin value public Map sessionOriginValue = new HashMap(); // check stmt is or not [select /*+ SET_VAR(...)*/ ...] @@ -472,18 +468,6 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = NUM_FREE_BLOCK_IN_SCAN) public int numFreeBlockInScan = 12; - /** - * For debugg purpose, dont' merge unique key and agg key when reading data. - */ - @VariableMgr.VarAttr(name = SKIP_STORAGE_ENGINE_MERGE) - public boolean skipStorageEngineMerge = false; - - /** - * For debugg purpose, skip delte predicate when reading data. - */ - @VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE) - public boolean skipDeletePredicate = false; - public String getBlockEncryptionMode() { return blockEncryptionMode; @@ -784,10 +768,6 @@ public void setShowHiddenColumns(boolean showHiddenColumns) { this.showHiddenColumns = showHiddenColumns; } - public boolean skipStorageEngineMerge() { - return skipStorageEngineMerge; - } - public boolean isAllowPartitionColumnNullable() { return allowPartitionColumnNullable; } @@ -997,10 +977,6 @@ public TQueryOptions toThrift() { tResult.setResourceLimit(resourceLimit); } - tResult.setSkipStorageEngineMerge(skipStorageEngineMerge); - - tResult.setSkipDeletePredicate(skipDeletePredicate); - return tResult; } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 900b904ea0785d..689cf07919e8b3 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -165,12 +165,6 @@ struct TQueryOptions { 44: optional bool trim_tailing_spaces_for_external_table_query = false 47: optional i32 num_free_block_in_scan - - // For debug purpose, dont' merge unique key and agg key when reading data. - 48: optional bool skip_storage_engine_merge = false - - // For debug purpose, skip delete predicates when reading data - 49: optional bool skip_delete_predicate = false } // A scan range plus the parameters needed to execute that scan.