[regression_test](debug) add regression test case for session variable skip_storage_engine_merge, skip_delete_predicate and show_hidden_columns

be/src/vec/exec/scan/new_olap_scanner.cpp

@@ -148,7 +148,12 @@ Status NewOlapScanner::_init_tablet_reader_params(
                       ->rowset_meta()
                       ->is_segments_overlapping());
 
-    _tablet_reader_params.direct_mode = _aggregation || single_version;
+    if (_state->skip_storage_engine_merge()) {
+        _tablet_reader_params.direct_mode = true;
+        _aggregation = true;
+    } else {
+        _tablet_reader_params.direct_mode = _aggregation || single_version;
+    }
 
     RETURN_IF_ERROR(_init_return_columns(!_tablet_reader_params.direct_mode));
 
@@ -169,10 +174,12 @@ Status NewOlapScanner::_init_tablet_reader_params(
     std::copy(function_filters.cbegin(), function_filters.cend(),
               std::inserter(_tablet_reader_params.function_filters,
                             _tablet_reader_params.function_filters.begin()));
-    auto& delete_preds = _tablet->delete_predicates();
-    std::copy(delete_preds.cbegin(), delete_preds.cend(),
-              std::inserter(_tablet_reader_params.delete_predicates,
-                            _tablet_reader_params.delete_predicates.begin()));
+    if (!_state->skip_delete_predicate()) {
+        auto& delete_preds = _tablet->delete_predicates();
+        std::copy(delete_preds.cbegin(), delete_preds.cend(),
+                  std::inserter(_tablet_reader_params.delete_predicates,
+                                _tablet_reader_params.delete_predicates.begin()));
+    }
 
     // Merge the columns in delete predicate that not in latest schema in to current tablet schema
     for (auto& del_pred_pb : _tablet_reader_params.delete_predicates) {
@@ -228,15 +235,17 @@ Status NewOlapScanner::_init_tablet_reader_params(
         _tablet_reader_params.delete_bitmap = &_tablet->tablet_meta()->delete_bitmap();
     }
 
-    TOlapScanNode& olap_scan_node = ((NewOlapScanNode*)_parent)->_olap_scan_node;
-    if (olap_scan_node.__isset.sort_info && olap_scan_node.sort_info.is_asc_order.size() > 0) {
-        _limit = _parent->_limit_per_scanner;
-        _tablet_reader_params.read_orderby_key = true;
-        if (!olap_scan_node.sort_info.is_asc_order[0]) {
-            _tablet_reader_params.read_orderby_key_reverse = true;
+    if (!_state->skip_storage_engine_merge()) {
+        TOlapScanNode& olap_scan_node = ((NewOlapScanNode*)_parent)->_olap_scan_node;
+        if (olap_scan_node.__isset.sort_info && olap_scan_node.sort_info.is_asc_order.size() > 0) {
+            _limit = _parent->_limit_per_scanner;
+            _tablet_reader_params.read_orderby_key = true;
+            if (!olap_scan_node.sort_info.is_asc_order[0]) {
+                _tablet_reader_params.read_orderby_key_reverse = true;
+            }
+            _tablet_reader_params.read_orderby_key_num_prefix_columns =
+                    olap_scan_node.sort_info.is_asc_order.size();
         }
-        _tablet_reader_params.read_orderby_key_num_prefix_columns =
-                olap_scan_node.sort_info.is_asc_order.size();
     }
 
     return Status::OK();

be/src/vec/exec/volap_scanner.cpp

@@ -173,6 +173,7 @@ Status VOlapScanner::_init_tablet_reader_params(
 
     if (_runtime_state->skip_storage_engine_merge()) {
         _tablet_reader_params.direct_mode = true;
+        _aggregation = true;
     } else {
         _tablet_reader_params.direct_mode = _aggregation || single_version;
     }
@@ -257,15 +258,17 @@ Status VOlapScanner::_init_tablet_reader_params(
         _tablet_reader_params.delete_bitmap = &_tablet->tablet_meta()->delete_bitmap();
     }
 
-    if (_parent->_olap_scan_node.__isset.sort_info &&
-        _parent->_olap_scan_node.sort_info.is_asc_order.size() > 0) {
-        _limit = _parent->_limit_per_scanner;
-        _tablet_reader_params.read_orderby_key = true;
-        if (!_parent->_olap_scan_node.sort_info.is_asc_order[0]) {
-            _tablet_reader_params.read_orderby_key_reverse = true;
+    if (!_runtime_state->skip_storage_engine_merge()) {
+        if (_parent->_olap_scan_node.__isset.sort_info &&
+            _parent->_olap_scan_node.sort_info.is_asc_order.size() > 0) {
+            _limit = _parent->_limit_per_scanner;
+            _tablet_reader_params.read_orderby_key = true;
+            if (!_parent->_olap_scan_node.sort_info.is_asc_order[0]) {
+                _tablet_reader_params.read_orderby_key_reverse = true;
+            }
+            _tablet_reader_params.read_orderby_key_num_prefix_columns =
+                    _parent->_olap_scan_node.sort_info.is_asc_order.size();
         }
-        _tablet_reader_params.read_orderby_key_num_prefix_columns =
-                _parent->_olap_scan_node.sort_info.is_asc_order.size();
     }
 
     return Status::OK();

regression-test/data/data_model_p0/unique/test_unique_table_debug_data.out

@@ -0,0 +1,34 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_init --
+1	11
+2	11
+3	1
+
+-- !select_skip_merge --
+1	1	0
+1	11	0
+2	1	0
+2	11	0
+3	1	0
+
+-- !select_batch_delete --
+2	11
+3	1
+
+-- !select_sql_delete --
+3	1
+
+-- !select_skip_merge_after_delete --
+1	1	0
+1	11	0
+1	111	1
+3	1	0
+
+-- !select_skip_delete --
+1	1	0
+1	11	0
+1	111	1
+2	1	0
+2	11	0
+3	1	0
+

regression-test/data/data_model_p0/unique/test_unique_table_debug_data_delete.csv

@@ -0,0 +1 @@
+1|111

regression-test/suites/data_model_p0/unique/test_unique_table_debug_data.groovy

@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_unique_table_debug_data") {
+
+    sql "ADMIN SET FRONTEND CONFIG ('enable_batch_delete_by_default' = 'true')"
+    sql "SET show_hidden_columns=false"
+    sql "SET skip_delete_predicate=false"
+    sql "SET skip_storage_engine_merge=false"
+
+    def tbName = "test_unique_table_debug_data"
+    sql "DROP TABLE IF EXISTS ${tbName}"
+    sql """
+            CREATE TABLE IF NOT EXISTS ${tbName} (
+                a int, b int
+            )
+            unique key (a)
+            distributed by hash(a) buckets 16
+            properties(
+                "replication_allocation" = "tag.location.default:1",
+                "disable_auto_compaction" = "false"
+            );
+        """
+
+    //BackendId,Cluster,IP,HeartbeatPort,BePort,HttpPort,BrpcPort,LastStartTime,LastHeartbeat,Alive,SystemDecommissioned,ClusterDecommissioned,TabletNum,DataUsedCapacity,AvailCapacity,TotalCapacity,UsedPct,MaxDiskUsedPct,Tag,ErrMsg,Version,Status
+    String[][] backends = sql """ show backends; """
+    assertTrue(backends.size() > 0)
+    StringBuilder sbCommand = new StringBuilder();
+
+    sql "insert into ${tbName} values(1,1),(2,1);"
+    sql "insert into ${tbName} values(1,11),(2,11);"
+    sql "insert into ${tbName} values(3,1);"
+
+    qt_select_init "select * from ${tbName} order by a, b"
+
+    // enable skip_storage_engine_merge and check select result,
+    // not merged original rows are returned:
+    sql "SET skip_storage_engine_merge=true"
+    qt_select_skip_merge "select * from ${tbName} order by a, b"
+
+    // turn off skip_storage_engine_merge
+    sql "SET skip_storage_engine_merge=false"
+
+    // batch delete and select again:
+    // curl --location-trusted -uroot: -H "column_separator:|" -H "columns:a, b" -H "merge_type: delete" -T delete.csv http://127.0.0.1:8030/api/test_skip/t1/_stream_load
+    streamLoad {
+        table "${tbName}"
+
+        set 'column_separator', '|'
+        set 'columns', 'a, b'
+        set 'merge_type', 'delete'
+
+        file 'test_unique_table_debug_data_delete.csv'
+
+        time 10000 // limit inflight 10s
+    }
+    qt_select_batch_delete "select * from ${tbName} order by a, b"
+
+    // delete rows with a = 2:
+    sql "delete from ${tbName} where a = 2;"
+    qt_select_sql_delete "select * from ${tbName} order by a, b"
+
+    // enable skip_storage_engine_merge and select, rows deleted with delete statement is not returned:
+    sql "SET skip_storage_engine_merge=true"
+    qt_select_skip_merge_after_delete "select * from ${tbName} order by a, b"
+
+    // enable skip_delete_predicate, rows deleted with delete statement is also returned:
+    sql "SET skip_delete_predicate=true"
+    qt_select_skip_delete "select * from ${tbName} order by a, b"
+
+    sql "DROP TABLE ${tbName}"
+}