From d4d2b06c1929fa0a5fbaf53abb677640a0604117 Mon Sep 17 00:00:00 2001 From: jacktengg <18241664+jacktengg@users.noreply.github.com> Date: Fri, 19 Aug 2022 15:09:55 +0800 Subject: [PATCH 1/3] [improvement](storage) For debugging problems: add session variable to treat agg and unique data model as dup model --- be/src/exec/olap_scanner.cpp | 6 +++++- be/src/runtime/runtime_state.h | 4 ++++ .../main/java/org/apache/doris/qe/SessionVariable.java | 10 ++++++++++ gensrc/thrift/PaloInternalService.thrift | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/be/src/exec/olap_scanner.cpp b/be/src/exec/olap_scanner.cpp index a5f29970eaf176..6b92f9813a531b 100644 --- a/be/src/exec/olap_scanner.cpp +++ b/be/src/exec/olap_scanner.cpp @@ -153,7 +153,11 @@ Status OlapScanner::_init_tablet_reader_params( ->rowset_meta() ->is_segments_overlapping()); - _tablet_reader_params.direct_mode = single_version || _aggregation; + if (_runtime_state->agg_as_duplicate()) { + _tablet_reader_params.direct_mode = true; + } else { + _tablet_reader_params.direct_mode = single_version || _aggregation; + } RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode)); diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 4f03a206e0adf0..2bbd8bd58ad5a9 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -357,6 +357,10 @@ class RuntimeState { return _query_options.enable_enable_exchange_node_parallel_merge; } + bool agg_as_duplicate() const { + return _query_options.agg_as_duplicate; + } + // the following getters are only valid after Prepare() InitialReservations* initial_reservations() const { return _initial_reservations; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 204ce8dbaa9063..df86e37bbce55c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -189,6 +189,8 @@ public class SessionVariable implements Serializable, Writable { static final String SESSION_CONTEXT = "session_context"; + static final String AGG_AS_DUPLICATE = "agg_as_duplicate"; + // session origin value public Map sessionOriginValue = new HashMap(); // check stmt is or not [select /*+ SET_VAR(...)*/ ...] @@ -463,6 +465,12 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = SESSION_CONTEXT, needForward = true) public String sessionContext = ""; + /** + * For debugg purpose, dont' merge unique key and agg key when reading data. + */ + @VariableMgr.VarAttr(name = AGG_AS_DUPLICATE) + private boolean aggAsDuplicate = false; + public String getBlockEncryptionMode() { return blockEncryptionMode; } @@ -970,6 +978,8 @@ public TQueryOptions toThrift() { tResult.setResourceLimit(resourceLimit); } + tResult.setAggAsDuplicate(aggAsDuplicate); + return tResult; } diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index aa1576b95a3419..142860551e0459 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -163,6 +163,9 @@ struct TQueryOptions { // trim tailing spaces while querying external table and stream load 44: optional bool trim_tailing_spaces_for_external_table_query = false + + // For debugg purpose, dont' merge unique key and agg key when reading data. + 45: optional bool agg_as_duplicate = false } From d6a2030aceda2bf12edb4c613a8fcbe24f98a4c1 Mon Sep 17 00:00:00 2001 From: jacktengg <18241664+jacktengg@users.noreply.github.com> Date: Fri, 19 Aug 2022 17:50:16 +0800 Subject: [PATCH 2/3] fix typo --- gensrc/thrift/PaloInternalService.thrift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 142860551e0459..27ac7259ebd856 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -164,7 +164,7 @@ struct TQueryOptions { // trim tailing spaces while querying external table and stream load 44: optional bool trim_tailing_spaces_for_external_table_query = false - // For debugg purpose, dont' merge unique key and agg key when reading data. + // For debug purpose, dont' merge unique key and agg key when reading data. 45: optional bool agg_as_duplicate = false } From 2cf82b49f24c0310271e9fc7d064cc6bae496588 Mon Sep 17 00:00:00 2001 From: jacktengg <18241664+jacktengg@users.noreply.github.com> Date: Sat, 20 Aug 2022 11:50:45 +0800 Subject: [PATCH 3/3] Update docs --- docs/en/administrator-guide/variables.md | 5 ++++- docs/zh-CN/administrator-guide/variables.md | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md index 3eb2ac31216272..c525b389ebd3be 100644 --- a/docs/en/administrator-guide/variables.md +++ b/docs/en/administrator-guide/variables.md @@ -496,4 +496,7 @@ Translated with www.DeepL.com/Translator (free version) SM4_128_CFB128, SM4_128_OFB, SM4_128_CTR, -``` \ No newline at end of file +``` + +* `agg_as_duplicate` + For debugging purpose. In case of problems of reading data of Aggregate Key model and Unique Key model, setting value to `true` will read data as Duplicate Key model. \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md index 219e86049e326c..6b39eb4e10d72b 100644 --- a/docs/zh-CN/administrator-guide/variables.md +++ b/docs/zh-CN/administrator-guide/variables.md @@ -490,3 +490,5 @@ SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3); 用于控制是否进行谓词推导。取值有两种：true 和 false。默认情况下关闭，系统不在进行谓词推导，采用原始的谓词进行相关操作。设置为 true 后，进行谓词扩展。 +* `agg_as_duplicate` + 用于调试目的。当发现读取Aggregate Key模型或者Unique Key模型的数据结果有问题的时候，把此变量的值设置为`true`，将会把Aggregate Key模型或者Unique Key模型的数据当成Duplicate Key模型读取。