[fix](index) Fix index for none key column of unique mor table

be/src/olap/rowset/beta_rowset_reader.cpp

@@ -161,10 +161,24 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
     }
 
     if (_read_context->predicates_except_leafnode_of_andnode != nullptr) {
-        _read_options.column_predicates_except_leafnode_of_andnode.insert(
-                _read_options.column_predicates_except_leafnode_of_andnode.end(),
-                _read_context->predicates_except_leafnode_of_andnode->begin(),
-                _read_context->predicates_except_leafnode_of_andnode->end());
+        bool should_push_down = true;
+        bool should_push_down_value_predicates = _should_push_down_value_predicates();
+        for (auto pred : *_read_context->predicates_except_leafnode_of_andnode) {
+            if (_rowset->keys_type() == UNIQUE_KEYS && !should_push_down_value_predicates &&
+                !_read_context->tablet_schema->column(pred->column_id()).is_key()) {
+                VLOG_DEBUG << "do not push down except_leafnode_of_andnode value pred "
+                           << pred->debug_string();
+                should_push_down = false;
+                break;
+            }
+        }
+
+        if (should_push_down) {
+            _read_options.column_predicates_except_leafnode_of_andnode.insert(
+                    _read_options.column_predicates_except_leafnode_of_andnode.end(),
+                    _read_context->predicates_except_leafnode_of_andnode->begin(),
+                    _read_context->predicates_except_leafnode_of_andnode->end());
+        }
     }
 
     // Take a delete-bitmap for each segment, the bitmap contains all deletes

be/src/olap/rowset/segment_v2/segment_iterator.cpp

@@ -459,12 +459,16 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() {
                  it != _remaining_conjunct_roots.end();) {
                 _pred_except_leafnode_of_andnode_evaluate_result.clear();
                 auto res = _execute_predicates_except_leafnode_of_andnode(*it);
+                VLOG_DEBUG << "_execute_predicates_except_leafnode_of_andnode expr: "
+                           << (*it)->debug_string() << " res: " << res;
                 if (res.ok() && _pred_except_leafnode_of_andnode_evaluate_result.size() == 1) {
                     _row_bitmap &= _pred_except_leafnode_of_andnode_evaluate_result[0];
                     // Delete expr after it obtains the final result.
                     {
                         std::erase_if(_common_expr_ctxs_push_down,
                                       [&it](const auto& iter) { return iter->root() == *it; });
+                        VLOG_DEBUG << "_remaining_conjunct_roots erase expr: "
+                                   << (*it)->debug_string();
                         it = _remaining_conjunct_roots.erase(it);
                     }
                 } else {
@@ -720,21 +724,24 @@ Status SegmentIterator::_execute_compound_fn(const std::string& function_name) {
     auto size = _pred_except_leafnode_of_andnode_evaluate_result.size();
     if (function_name == "and") {
         if (size < 2) {
-            return Status::InternalError("execute and logic compute error.");
+            return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name,
+                                           size);
         }
         _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) &=
                 _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1);
         _pred_except_leafnode_of_andnode_evaluate_result.pop_back();
     } else if (function_name == "or") {
         if (size < 2) {
-            return Status::InternalError("execute or logic compute error.");
+            return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 2", function_name,
+                                           size);
         }
         _pred_except_leafnode_of_andnode_evaluate_result.at(size - 2) |=
                 _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1);
         _pred_except_leafnode_of_andnode_evaluate_result.pop_back();
     } else if (function_name == "not") {
         if (size < 1) {
-            return Status::InternalError("execute not logic compute error.");
+            return Status::InvalidArgument("_execute_compound_fn {} arg num {} < 1", function_name,
+                                           size);
         }
         roaring::Roaring tmp = _row_bitmap;
         tmp -= _pred_except_leafnode_of_andnode_evaluate_result.at(size - 1);

fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java

@@ -224,12 +224,9 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe
                 throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. "
                         + "invalid index: " + indexName);
             }
-            if (!column.isKey()
-                    && ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite)
-                        || keysType == KeysType.AGG_KEYS)) {
-                throw new AnalysisException(indexType.toString()
-                    + " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table."
-                    + " invalid index: " + indexName);
+            if (!column.isKey() && keysType == KeysType.AGG_KEYS) {
+                throw new AnalysisException("index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table"
+                    + " or key columns of AGG_KEYS table. invalid index: " + indexName);
             }
 
             if (indexType == IndexType.INVERTED) {

fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java

@@ -466,8 +466,8 @@ public static Set<String> analyzeBloomFilterColumns(Map<String, String> properti
                             found = true;
                             break;
                         } else {
-                            throw new AnalysisException("Bloom filter index only used in columns of"
-                                    + " UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table."
+                            throw new AnalysisException("Bloom filter index should only be used in columns"
+                                    + " of UNIQUE_KEYS/DUP_KEYS table or key columns of AGG_KEYS table."
                                     + " invalid column: " + bfColumn);
                         }
                     }

fe/fe-core/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java

@@ -122,7 +122,7 @@ public void testBfColumnsError() {
         try {
             PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns, KeysType.AGG_KEYS);
         } catch (AnalysisException e) {
-            Assert.assertTrue(e.getMessage().contains("Bloom filter index only used in"));
+            Assert.assertTrue(e.getMessage().contains("Bloom filter index should only be used"));
         }
 
         // reduplicated column

regression-test/data/inverted_index_p0/test_inverted_index_mor.out

@@ -0,0 +1,30 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !11 --
+1	2	12	1.2	1 2
+3	4	34	3.4	3 4
+11	12	1112	11.12	11 22
+13	14	1314	13.14	13 14
+
+-- !12 --
+1	2	12	1.2	1 2
+3	4	34	3.4	3 4
+11	12	1112	11.12	11 22
+13	14	1314	13.14	13 14
+
+-- !21 --
+1	2	112	11.2	112
+3	4	34	3.4	3 4
+11	12	1112	11.12	11 22
+13	14	1314	13.14	13 14
+
+-- !22 --
+3	4	34	3.4	3 4
+11	12	1112	11.12	11 22
+13	14	1314	13.14	13 14
+
+-- !23 --
+1	2	112	11.2	112
+3	4	34	3.4	3 4
+11	12	1112	11.12	11 22
+13	14	1314	13.14	13 14
+

regression-test/suites/datatype_p0/scalar_types/load.groovy

@@ -305,6 +305,60 @@ suite("test_scalar_types_load", "p0") {
             `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`,
             `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup"""
 
+    // define unique key table2 enable mow
+    testTable = "tbl_scalar_types_unique2_bitmapindex"
+    sql "DROP TABLE IF EXISTS ${testTable}"
+    sql """
+        CREATE TABLE IF NOT EXISTS ${testTable} (
+            `c_datetimev2` datetimev2(0) NULL,
+            `c_bigint` bigint(20) NULL,
+            `c_decimalv3` decimalv3(20, 3) NULL,
+            `c_bool` boolean NULL,
+            `c_tinyint` tinyint(4) NULL,
+            `c_smallint` smallint(6) NULL,
+            `c_int` int(11) NULL,
+            `c_largeint` largeint(40) NULL,
+            `c_float` float NULL,
+            `c_double` double NULL,
+            `c_decimal` decimal(20, 3) NULL,
+            `c_date` date NULL,
+            `c_datetime` datetime NULL,
+            `c_datev2` datev2 NULL,
+            `c_char` char(15) NULL,
+            `c_varchar` varchar(100) NULL,
+            `c_string` text NULL,
+            INDEX idx_c_bool (c_bool) USING BITMAP,
+            INDEX idx_c_tinyint (c_tinyint) USING BITMAP,
+            INDEX idx_c_smallint (c_smallint) USING BITMAP,
+            INDEX idx_c_int (c_int) USING BITMAP,
+            INDEX idx_c_bigint (c_bigint) USING BITMAP,
+            INDEX idx_c_largeint (c_largeint) USING BITMAP,
+            INDEX idx_c_decimal (c_decimal) USING BITMAP,
+            INDEX idx_c_decimalv3 (c_decimalv3) USING BITMAP,
+            INDEX idx_c_date (c_date) USING BITMAP,
+            INDEX idx_c_datetime (c_datetime) USING BITMAP,
+            INDEX idx_c_datev2 (c_datev2) USING BITMAP,
+            INDEX idx_c_datetimev2 (c_datetimev2) USING BITMAP,
+            INDEX idx_c_char (c_char) USING BITMAP,
+            INDEX idx_c_varchar (c_varchar) USING BITMAP,
+            INDEX idx_c_string (c_string) USING BITMAP
+        ) ENGINE=OLAP
+        UNIQUE KEY(`c_datetimev2`, `c_bigint`, `c_decimalv3`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`c_bigint`) BUCKETS 10
+        PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "false");
+        """
+
+    // insert data into unique key table2 2 times
+    sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`,
+            `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`,
+            `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`,
+            `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup"""
+    sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`,
+            `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`,
+            `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`,
+            `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup"""
+
 
     // define dup key table with index
     testTable = "tbl_scalar_types_dup_inverted_index"

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql

@@ -0,0 +1 @@
+SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bool = 1; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql

@@ -0,0 +1 @@
+SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_tinyint = 14; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_smallint = 12321 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_int = -16441 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bigint = 859063837 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_largeint = 1524137702 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_float = -13023.051 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_double = -679666086.055166 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimal = 5355625357307892.981 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimalv3 = 89599956401481176.347 ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_date = '2022-10-09' ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetime = '2022-07-31 12:07:00' ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datev2 = '2022-03-06' ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetimev2 = '2022-11-18 06:05:56' ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_char = '202.171.74.217' ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_varchar = 'modi_a@Feedmix.biz' ORDER BY c_bigint, c_largeint; 

regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql

@@ -0,0 +1 @@
+SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_string = 'Reinke Junction 63' ORDER BY c_bigint, c_largeint; 

regression-test/suites/index_p0/test_bitmap_index.groovy

@@ -175,7 +175,7 @@ suite("test_bitmap_index") {
     }
     test{
         sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING BITMAP;"
-        exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16"
+        exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index16"
     }
 
     sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);"

regression-test/suites/inverted_index_p0/test_bitmap_index.groovy

@@ -155,7 +155,7 @@ suite("test_bitmap_index", "inverted_index") {
     }
     test{
         sql "ALTER TABLE ${tbName2} ADD INDEX index12 (v1) USING BITMAP;"
-        exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index12"
+        exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index12"
     }
 
     sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,1);"

regression-test/suites/inverted_index_p0/test_inverted_index.groovy

@@ -175,7 +175,7 @@ suite("test_inverted_index", "inverted_index") {
     }
     test{
         sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING INVERTED;"
-        exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16"
+        exception "errCode = 2, detailMessage = index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid index: index16"
     }
 
     sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);"

regression-test/suites/inverted_index_p0/test_inverted_index_mor.groovy

@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_inverted_index_mor", "p0"){
+    def timeout = 60000
+    def delta_time = 1000
+    def alter_res = "null"
+    def useTime = 0
+
+    def indexTblName = "test_inverted_index_mor"
+
+    sql "DROP TABLE IF EXISTS ${indexTblName}"
+
+    sql """
+	CREATE TABLE IF NOT EXISTS ${indexTblName}(
+	    `k1` INT,
+        `k2` INT,
+        `c_int` INT,
+        `c_float` FLOAT,
+        `c_string` STRING,
+        INDEX idx_k2(`k2`) USING INVERTED COMMENT '',
+        INDEX idx_c_int(`c_int`) USING INVERTED COMMENT '',
+        INDEX idx_c_string(`c_string`) USING INVERTED PROPERTIES("parser"="english") COMMENT ''
+	) ENGINE=OLAP
+	UNIQUE KEY(`k1`, `k2`)
+	COMMENT 'OLAP'
+	DISTRIBUTED BY HASH(`k1`, `k2`) BUCKETS 1
+	PROPERTIES(
+ 	    "replication_allocation" = "tag.location.default: 1",
+        "enable_unique_key_merge_on_write" = "false",
+        "disable_auto_compaction" = "true"
+	);
+    """
+
+    sql """ INSERT INTO $indexTblName VALUES (1, 2, 12, 1.2, '1 2'), (3, 4, 34, 3.4, '3 4'); """
+    sql """ INSERT INTO $indexTblName VALUES (11, 12, 1112, 11.12, '11 22'), (13, 14, 1314, 13.14, '13 14'); """
+
+    // original data
+    qt_11 """ SELECT * FROM $indexTblName ORDER BY k1,k2 """
+
+    //
+    qt_12 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 12 ORDER BY k1,k2 """
+
+    sql """ INSERT INTO $indexTblName VALUES (1, 2, 112, 11.2, '112'); """
+    qt_21 """ SELECT * FROM $indexTblName ORDER BY k1,k2 """
+    qt_22 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 12 ORDER BY k1,k2 """
+    qt_23 """ SELECT * FROM $indexTblName WHERE k2 > 2 OR c_int = 112 ORDER BY k1,k2 """
+}

regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy

@@ -68,7 +68,7 @@ suite("test_segcompaction_unique_keys_index") {
                 UNIQUE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1
                 PROPERTIES ( "replication_num" = "1" );
             """
-            exception "INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid indexName: index_col_1"
+            exception "index should only be used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid indexName: index_col_1"
         }
     } finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")