diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 8b49f0a161766f..1b7042815a37fb 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -349,6 +349,15 @@ class RuntimeState { return segment_v2::CompressionTypePB::SNAPPY; } + bool skip_storage_engine_merge() const { + return _query_options.__isset.skip_storage_engine_merge && + _query_options.skip_storage_engine_merge; + } + + bool skip_delete_predicate() const { + return _query_options.__isset.skip_delete_predicate && _query_options.skip_delete_predicate; + } + const std::vector& tablet_commit_infos() const { return _tablet_commit_infos; } diff --git a/be/src/vec/exec/volap_scanner.cpp b/be/src/vec/exec/volap_scanner.cpp index efa80b36ac0903..4b820aa1cec057 100644 --- a/be/src/vec/exec/volap_scanner.cpp +++ b/be/src/vec/exec/volap_scanner.cpp @@ -171,7 +171,11 @@ Status VOlapScanner::_init_tablet_reader_params( ->rowset_meta() ->is_segments_overlapping()); - _tablet_reader_params.direct_mode = _aggregation || single_version; + if (_runtime_state->skip_storage_engine_merge()) { + _tablet_reader_params.direct_mode = true; + } else { + _tablet_reader_params.direct_mode = _aggregation || single_version; + } RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode)); @@ -192,10 +196,12 @@ Status VOlapScanner::_init_tablet_reader_params( std::copy(function_filters.cbegin(), function_filters.cend(), std::inserter(_tablet_reader_params.function_filters, _tablet_reader_params.function_filters.begin())); - auto& delete_preds = _tablet->delete_predicates(); - std::copy(delete_preds.cbegin(), delete_preds.cend(), - std::inserter(_tablet_reader_params.delete_predicates, - _tablet_reader_params.delete_predicates.begin())); + if (!_runtime_state->skip_delete_predicate()) { + auto& delete_preds = _tablet->delete_predicates(); + std::copy(delete_preds.cbegin(), delete_preds.cend(), + std::inserter(_tablet_reader_params.delete_predicates, + _tablet_reader_params.delete_predicates.begin())); + } // Merge the columns in delete predicate that not in latest schema in to current tablet schema for (auto& del_pred_rs : _tablet_reader_params.delete_predicates) { diff --git a/docs/en/docs/advanced/variables.md b/docs/en/docs/advanced/variables.md index 5b388c8a053165..537a271e73869f 100644 --- a/docs/en/docs/advanced/variables.md +++ b/docs/en/docs/advanced/variables.md @@ -526,3 +526,9 @@ Translated with www.DeepL.com/Translator (free version) * `trim_tailing_spaces_for_external_table_query` Used to control whether trim the tailing spaces while quering Hive external tables. The default is false. + +* `skip_storage_engine_merge` + For debugging purpose. In vectorized execution engine, in case of problems of reading data of Aggregate Key model and Unique Key model, setting value to `true` will read data as Duplicate Key model. + +* `skip_delete_predicate` + For debugging purpose. In vectorized execution engine, in case of problems of reading data, setting value to `true` will also read deleted data. diff --git a/docs/zh-CN/docs/advanced/variables.md b/docs/zh-CN/docs/advanced/variables.md index c548f452f90bda..6ce5eb836e8d56 100644 --- a/docs/zh-CN/docs/advanced/variables.md +++ b/docs/zh-CN/docs/advanced/variables.md @@ -515,3 +515,9 @@ SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3); - `trim_tailing_spaces_for_external_table_query` 用于控制查询Hive外表时是否过滤掉字段末尾的空格。默认为false。 + +* `skip_storage_engine_merge` + 用于调试目的。在向量化执行引擎中,当发现读取Aggregate Key模型或者Unique Key模型的数据结果有问题的时候,把此变量的值设置为`true`,将会把Aggregate Key模型或者Unique Key模型的数据当成Duplicate Key模型读取。 + +* `skip_delete_predicate` + 用于调试目的。在向量化执行引擎中,当发现读取表的数据结果有误的时候,把此变量的值设置为`true`,将会把被删除的数据当成正常数据读取。 \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 0d5cf4eed7dc7a..f1f6fc62826281 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -455,7 +455,9 @@ public static InputStream getInputStreamFromUrl(String urlStr, String encodedAut } public static boolean showHiddenColumns() { - return ConnectContext.get() != null && ConnectContext.get().getSessionVariable().showHiddenColumns(); + return ConnectContext.get() != null && ( + ConnectContext.get().getSessionVariable().showHiddenColumns() + || ConnectContext.get().getSessionVariable().skipStorageEngineMerge()); } public static String escapeSingleRegex(String s) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index da79ae041095c7..a12ee9059c43a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -211,6 +211,10 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_LOCAL_EXCHANGE = "enable_local_exchange"; + public static final String SKIP_STORAGE_ENGINE_MERGE = "skip_storage_engine_merge"; + + public static final String SKIP_DELETE_PREDICATE = "skip_delete_predicate"; + // session origin value public Map sessionOriginValue = new HashMap(); // check stmt is or not [select /*+ SET_VAR(...)*/ ...] @@ -530,6 +534,18 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_LOCAL_EXCHANGE) public boolean enableLocalExchange = false; + /** + * For debugg purpose, dont' merge unique key and agg key when reading data. + */ + @VariableMgr.VarAttr(name = SKIP_STORAGE_ENGINE_MERGE) + public boolean skipStorageEngineMerge = false; + + /** + * For debugg purpose, skip delte predicate when reading data. + */ + @VariableMgr.VarAttr(name = SKIP_DELETE_PREDICATE) + public boolean skipDeletePredicate = false; + public String getBlockEncryptionMode() { return blockEncryptionMode; } @@ -843,6 +859,10 @@ public void setShowHiddenColumns(boolean showHiddenColumns) { this.showHiddenColumns = showHiddenColumns; } + public boolean skipStorageEngineMerge() { + return skipStorageEngineMerge; + } + public boolean isAllowPartitionColumnNullable() { return allowPartitionColumnNullable; } @@ -1124,6 +1144,10 @@ public TQueryOptions toThrift() { tResult.setFragmentTransmissionCompressionCodec(fragmentTransmissionCompressionCodec); tResult.setEnableLocalExchange(enableLocalExchange); + tResult.setSkipStorageEngineMerge(skipStorageEngineMerge); + + tResult.setSkipDeletePredicate(skipDeletePredicate); + return tResult; } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index c1c9f97d07a665..d9911e61411c80 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -169,6 +169,12 @@ struct TQueryOptions { 46: optional string fragment_transmission_compression_codec; 47: optional bool enable_local_exchange; + + // For debug purpose, dont' merge unique key and agg key when reading data. + 48: optional bool skip_storage_engine_merge = false + + // For debug purpose, skip delete predicates when reading data + 49: optional bool skip_delete_predicate = false }