[FIX] Pick array inverted index bugfix

be/src/exec/olap_common.h

@@ -318,16 +318,6 @@ class ColumnValueRange {
                 condition.__set_condition_op("match_regexp");
             } else if (value.first == MatchType::MATCH_PHRASE_EDGE) {
                 condition.__set_condition_op("match_phrase_edge");
-            } else if (value.first == MatchType::MATCH_ELEMENT_EQ) {
-                condition.__set_condition_op("match_element_eq");
-            } else if (value.first == MatchType::MATCH_ELEMENT_LT) {
-                condition.__set_condition_op("match_element_lt");
-            } else if (value.first == MatchType::MATCH_ELEMENT_GT) {
-                condition.__set_condition_op("match_element_gt");
-            } else if (value.first == MatchType::MATCH_ELEMENT_LE) {
-                condition.__set_condition_op("match_element_le");
-            } else if (value.first == MatchType::MATCH_ELEMENT_GE) {
-                condition.__set_condition_op("match_element_ge");
             }
             condition.condition_values.push_back(
                     cast_to_string<primitive_type, CppType>(value.second, _scale));

be/src/exec/olap_utils.h

@@ -132,11 +132,6 @@ enum class MatchType {
     MATCH_ANY = 0,
     MATCH_ALL = 1,
     MATCH_PHRASE = 2,
-    MATCH_ELEMENT_EQ = 3,
-    MATCH_ELEMENT_LT = 4,
-    MATCH_ELEMENT_GT = 5,
-    MATCH_ELEMENT_LE = 6,
-    MATCH_ELEMENT_GE = 7,
     MATCH_PHRASE_PREFIX = 8,
     MATCH_REGEXP = 9,
     MATCH_PHRASE_EDGE = 10,
@@ -162,21 +157,6 @@ inline MatchType to_match_type(TExprOpcode::type type) {
     case TExprOpcode::type::MATCH_PHRASE_EDGE:
         return MatchType::MATCH_PHRASE_EDGE;
         break;
-    case TExprOpcode::type::MATCH_ELEMENT_EQ:
-        return MatchType::MATCH_ELEMENT_EQ;
-        break;
-    case TExprOpcode::type::MATCH_ELEMENT_LT:
-        return MatchType::MATCH_ELEMENT_LT;
-        break;
-    case TExprOpcode::type::MATCH_ELEMENT_GT:
-        return MatchType::MATCH_ELEMENT_GT;
-        break;
-    case TExprOpcode::type::MATCH_ELEMENT_LE:
-        return MatchType::MATCH_ELEMENT_LE;
-        break;
-    case TExprOpcode::type::MATCH_ELEMENT_GE:
-        return MatchType::MATCH_ELEMENT_GE;
-        break;
     default:
         VLOG_CRITICAL << "TExprOpcode: " << type;
         DCHECK(false);
@@ -197,16 +177,6 @@ inline MatchType to_match_type(const std::string& condition_op) {
         return MatchType::MATCH_REGEXP;
     } else if (condition_op.compare("match_phrase_edge") == 0) {
         return MatchType::MATCH_PHRASE_EDGE;
-    } else if (condition_op.compare("match_element_eq") == 0) {
-        return MatchType::MATCH_ELEMENT_EQ;
-    } else if (condition_op.compare("match_element_lt") == 0) {
-        return MatchType::MATCH_ELEMENT_LT;
-    } else if (condition_op.compare("match_element_gt") == 0) {
-        return MatchType::MATCH_ELEMENT_GT;
-    } else if (condition_op.compare("match_element_le") == 0) {
-        return MatchType::MATCH_ELEMENT_LE;
-    } else if (condition_op.compare("match_element_ge") == 0) {
-        return MatchType::MATCH_ELEMENT_GE;
     }
     return MatchType::UNKNOWN;
 }
@@ -216,12 +186,7 @@ inline bool is_match_condition(const std::string& op) {
         0 == strcasecmp(op.c_str(), "match_phrase") ||
         0 == strcasecmp(op.c_str(), "match_phrase_prefix") ||
         0 == strcasecmp(op.c_str(), "match_regexp") ||
-        0 == strcasecmp(op.c_str(), "match_phrase_edge") ||
-        0 == strcasecmp(op.c_str(), "match_element_eq") ||
-        0 == strcasecmp(op.c_str(), "match_element_lt") ||
-        0 == strcasecmp(op.c_str(), "match_element_gt") ||
-        0 == strcasecmp(op.c_str(), "match_element_le") ||
-        0 == strcasecmp(op.c_str(), "match_element_ge")) {
+        0 == strcasecmp(op.c_str(), "match_phrase_edge")) {
         return true;
     }
     return false;
@@ -230,10 +195,7 @@ inline bool is_match_condition(const std::string& op) {
 inline bool is_match_operator(const TExprOpcode::type& op_type) {
     return TExprOpcode::MATCH_ANY == op_type || TExprOpcode::MATCH_ALL == op_type ||
            TExprOpcode::MATCH_PHRASE == op_type || TExprOpcode::MATCH_PHRASE_PREFIX == op_type ||
-           TExprOpcode::MATCH_REGEXP == op_type || TExprOpcode::MATCH_PHRASE_EDGE == op_type ||
-           TExprOpcode::MATCH_ELEMENT_EQ == op_type || TExprOpcode::MATCH_ELEMENT_LT == op_type ||
-           TExprOpcode::MATCH_ELEMENT_GT == op_type || TExprOpcode::MATCH_ELEMENT_LE == op_type ||
-           TExprOpcode::MATCH_ELEMENT_GE == op_type;
+           TExprOpcode::MATCH_REGEXP == op_type || TExprOpcode::MATCH_PHRASE_EDGE == op_type;
 }
 
 } // namespace doris

be/src/olap/match_predicate.cpp

@@ -116,21 +116,6 @@ InvertedIndexQueryType MatchPredicate::_to_inverted_index_query_type(MatchType m
     case MatchType::MATCH_PHRASE_EDGE:
         ret = InvertedIndexQueryType::MATCH_PHRASE_EDGE_QUERY;
         break;
-    case MatchType::MATCH_ELEMENT_EQ:
-        ret = InvertedIndexQueryType::EQUAL_QUERY;
-        break;
-    case MatchType::MATCH_ELEMENT_LT:
-        ret = InvertedIndexQueryType::LESS_THAN_QUERY;
-        break;
-    case MatchType::MATCH_ELEMENT_GT:
-        ret = InvertedIndexQueryType::GREATER_THAN_QUERY;
-        break;
-    case MatchType::MATCH_ELEMENT_LE:
-        ret = InvertedIndexQueryType::LESS_EQUAL_QUERY;
-        break;
-    case MatchType::MATCH_ELEMENT_GE:
-        ret = InvertedIndexQueryType::GREATER_EQUAL_QUERY;
-        break;
     default:
         DCHECK(false);
     }

be/src/olap/rowset/segment_v2/inverted_index_writer.cpp

@@ -374,16 +374,15 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
             auto ignore_above_value =
                     get_parser_ignore_above_value_from_properties(_index_meta->properties());
             auto ignore_above = std::stoi(ignore_above_value);
+            size_t start_off = 0;
             for (int i = 0; i < count; ++i) {
-                // offsets[i+1] is now row element count
-                // [0, 3, 6]
-                // [10,20,30] [20,30,40], [30,40,50]
-                auto start_off = offsets[i];
-                auto end_off = offsets[i + 1];
+                // nullmap & value ptr-array may not from offsets[i] because olap_convertor make offsets accumulate from _base_offset which may not is 0, but nullmap & value in this segment is from 0, we only need
+                // every single array row element size to go through the nullmap & value ptr-array, and also can go through the every row in array to keep with _rid++
+                auto array_elem_size = offsets[i + 1] - offsets[i];
                 // TODO(Amory).later we use object pool to avoid field creation
                 lucene::document::Field* new_field = nullptr;
                 CL_NS(analysis)::TokenStream* ts = nullptr;
-                for (auto j = start_off; j < end_off; ++j) {
+                for (auto j = start_off; j < start_off + array_elem_size; ++j) {
                     if (null_map[j] == 1) {
                         continue;
                     }
@@ -406,29 +405,46 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                             _parser_type != InvertedIndexParserType::PARSER_NONE) {
                             // in this case stream need to delete after add_document, because the
                             // stream can not reuse for different field
-                            _char_string_reader->init(v->get_data(), v->get_size(), false);
+                            std::unique_ptr<lucene::util::Reader> char_string_reader = nullptr;
+                            RETURN_IF_ERROR(create_char_string_reader(char_string_reader));
+                            char_string_reader->init(v->get_data(), v->get_size(), false);
                             ts = _analyzer->tokenStream(new_field->name(),
-                                                        _char_string_reader.get());
+                                                        char_string_reader.release());
                             new_field->setValue(ts);
                         } else {
                             new_field_char_value(v->get_data(), v->get_size(), new_field);
                         }
                         _doc->add(*new_field);
                     }
                 }
+                start_off += array_elem_size;
                 if (!_doc->getFields()->empty()) {
                     // if this array is null, we just ignore to write inverted index
                     RETURN_IF_ERROR(add_document());
                     _doc->clear();
                     _CLDELETE(ts);
+                } else {
+                    // avoid to add doc which without any field which may make threadState init skip
+                    // init fieldDataArray, then will make error with next doc with fields in
+                    // resetCurrentFieldData
+                    if (Status st = create_field(&new_field); st != Status::OK()) {
+                        LOG(ERROR)
+                                << "create field " << string(_field_name.begin(), _field_name.end())
+                                << " error:" << st;
+                        return st;
+                    }
+                    _doc->add(*new_field);
+                    RETURN_IF_ERROR(add_null_document());
+                    _doc->clear();
+                    _CLDELETE(ts);
                 }
                 _rid++;
             }
         } else if constexpr (field_is_numeric_type(field_type)) {
+            size_t start_off = 0;
             for (int i = 0; i < count; ++i) {
-                auto start_off = offsets[i];
-                auto end_off = offsets[i + 1];
-                for (size_t j = start_off; j < end_off; ++j) {
+                auto array_elem_size = offsets[i + 1] - offsets[i];
+                for (size_t j = start_off; j < start_off + array_elem_size; ++j) {
                     if (null_map[j] == 1) {
                         continue;
                     }
@@ -439,6 +455,7 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                     _value_key_coder->full_encode_ascending(p, &new_value);
                     _bkd_writer->add((const uint8_t*)new_value.c_str(), value_length, _rid);
                 }
+                start_off += array_elem_size;
                 _row_ids_seen_for_bkd++;
                 _rid++;
             }

be/src/vec/functions/match.cpp

@@ -400,11 +400,6 @@ void register_function_match(SimpleFunctionFactory& factory) {
     factory.register_function<FunctionMatchPhrasePrefix>();
     factory.register_function<FunctionMatchRegexp>();
     factory.register_function<FunctionMatchPhraseEdge>();
-    factory.register_function<FunctionMatchElementEQ>();
-    factory.register_function<FunctionMatchElementLT>();
-    factory.register_function<FunctionMatchElementGT>();
-    factory.register_function<FunctionMatchElementLE>();
-    factory.register_function<FunctionMatchElementGE>();
 }
 
 } // namespace doris::vectorized

be/src/vec/functions/match.h

@@ -176,89 +176,4 @@ class FunctionMatchPhraseEdge : public FunctionMatchBase {
     }
 };
 
-class FunctionMatchElementEQ : public FunctionMatchBase {
-public:
-    static constexpr auto name = "match_element_eq";
-    static FunctionPtr create() { return std::make_shared<FunctionMatchElementEQ>(); }
-
-    String get_name() const override { return name; }
-
-    Status execute_match(const std::string& column_name, const std::string& match_query_str,
-                         size_t input_rows_count, const ColumnString* string_col,
-                         InvertedIndexCtx* inverted_index_ctx,
-                         const ColumnArray::Offsets64* array_offsets,
-                         ColumnUInt8::Container& result) const override {
-        return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
-                "FunctionMatchElementEQ not support execute_match");
-    }
-};
-
-class FunctionMatchElementLT : public FunctionMatchBase {
-public:
-    static constexpr auto name = "match_element_lt";
-    static FunctionPtr create() { return std::make_shared<FunctionMatchElementLT>(); }
-
-    String get_name() const override { return name; }
-
-    Status execute_match(const std::string& column_name, const std::string& match_query_str,
-                         size_t input_rows_count, const ColumnString* string_col,
-                         InvertedIndexCtx* inverted_index_ctx,
-                         const ColumnArray::Offsets64* array_offsets,
-                         ColumnUInt8::Container& result) const override {
-        return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
-                "FunctionMatchElementLT not support execute_match");
-    }
-};
-
-class FunctionMatchElementGT : public FunctionMatchBase {
-public:
-    static constexpr auto name = "match_element_gt";
-    static FunctionPtr create() { return std::make_shared<FunctionMatchElementGT>(); }
-
-    String get_name() const override { return name; }
-
-    Status execute_match(const std::string& column_name, const std::string& match_query_str,
-                         size_t input_rows_count, const ColumnString* string_col,
-                         InvertedIndexCtx* inverted_index_ctx,
-                         const ColumnArray::Offsets64* array_offsets,
-                         ColumnUInt8::Container& result) const override {
-        return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
-                "FunctionMatchElementGT not support execute_match");
-    }
-};
-
-class FunctionMatchElementLE : public FunctionMatchBase {
-public:
-    static constexpr auto name = "match_element_le";
-    static FunctionPtr create() { return std::make_shared<FunctionMatchElementLE>(); }
-
-    String get_name() const override { return name; }
-
-    Status execute_match(const std::string& column_name, const std::string& match_query_str,
-                         size_t input_rows_count, const ColumnString* string_col,
-                         InvertedIndexCtx* inverted_index_ctx,
-                         const ColumnArray::Offsets64* array_offsets,
-                         ColumnUInt8::Container& result) const override {
-        return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
-                "FunctionMatchElementLE not support execute_match");
-    }
-};
-
-class FunctionMatchElementGE : public FunctionMatchBase {
-public:
-    static constexpr auto name = "match_element_ge";
-    static FunctionPtr create() { return std::make_shared<FunctionMatchElementGE>(); }
-
-    String get_name() const override { return name; }
-
-    Status execute_match(const std::string& column_name, const std::string& match_query_str,
-                         size_t input_rows_count, const ColumnString* string_col,
-                         InvertedIndexCtx* inverted_index_ctx,
-                         const ColumnArray::Offsets64* array_offsets,
-                         ColumnUInt8::Container& result) const override {
-        return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
-                "FunctionMatchElementGE not support execute_match");
-    }
-};
-
 } // namespace doris::vectorized

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -2672,6 +2672,9 @@ public static boolean isNotCloudMode() {
     @ConfField(mutable = true)
     public static int cloud_cold_read_percent = 10; // 10%
 
+    @ConfField(mutable = true)
+    public static boolean enable_create_inverted_index_for_array = false;
+
     @ConfField(mutable = true, masterOnly = true)
     public static boolean enable_create_bitmap_index_as_inverted_index = true;
 

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4

@@ -351,11 +351,6 @@ MAP: 'MAP';
 MATCH: 'MATCH';
 MATCH_ALL: 'MATCH_ALL';
 MATCH_ANY: 'MATCH_ANY';
-MATCH_ELEMENT_EQ: 'ELEMENT_EQ';
-MATCH_ELEMENT_GE: 'ELEMENT_GE';
-MATCH_ELEMENT_GT: 'ELEMENT_GT';
-MATCH_ELEMENT_LE: 'ELEMENT_LE';
-MATCH_ELEMENT_LT: 'ELEMENT_LT';
 MATCH_PHRASE: 'MATCH_PHRASE';
 MATCH_PHRASE_EDGE: 'MATCH_PHRASE_EDGE';
 MATCH_PHRASE_PREFIX: 'MATCH_PHRASE_PREFIX';

fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4

@@ -1178,7 +1178,7 @@ nonReserved
     | MAP
     | MATCH_ALL
     | MATCH_ANY
-    | MATCH_ELEMENT_EQ
+    | MATCH_columnName
     | MATCH_ELEMENT_GE
     | MATCH_ELEMENT_GT
     | MATCH_ELEMENT_LE

fe/fe-core/src/main/cup/sql_parser.cup

@@ -489,11 +489,6 @@ terminal String
     KW_MATCH_PHRASE_PREFIX,
     KW_MATCH_REGEXP,
     KW_MATCH_PHRASE_EDGE,
-    KW_MATCH_ELEMENT_EQ,
-    KW_MATCH_ELEMENT_LT,
-    KW_MATCH_ELEMENT_GT,
-    KW_MATCH_ELEMENT_LE,
-    KW_MATCH_ELEMENT_GE,
     KW_NAME,
     KW_NAMES,
     KW_NATURAL,
@@ -1001,7 +996,7 @@ precedence left KW_AND;
 precedence left KW_NOT, NOT;
 precedence left KW_BETWEEN, KW_IN, KW_IS, KW_EXISTS;
 precedence left KW_LIKE, KW_REGEXP;
-precedence left KW_MATCH_ANY, KW_MATCH_ALL, KW_MATCH_PHRASE, KW_MATCH_PHRASE_PREFIX, KW_MATCH_REGEXP, KW_MATCH_PHRASE_EDGE, KW_MATCH, KW_MATCH_ELEMENT_EQ, KW_MATCH_ELEMENT_LT, KW_MATCH_ELEMENT_GT, KW_MATCH_ELEMENT_LE, KW_MATCH_ELEMENT_GE;
+precedence left KW_MATCH_ANY, KW_MATCH_ALL, KW_MATCH_PHRASE, KW_MATCH_PHRASE_PREFIX, KW_MATCH_REGEXP, KW_MATCH_PHRASE_EDGE, KW_MATCH;
 precedence left EQUAL, LESSTHAN, GREATERTHAN;
 precedence left ADD, SUBTRACT;
 precedence left AT, STAR, DIVIDE, MOD, KW_DIV;
@@ -7321,16 +7316,6 @@ match_predicate ::=
   {: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_REGEXP, e1, e2); :}
   | expr:e1 KW_MATCH_PHRASE_EDGE expr:e2
   {: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_PHRASE_EDGE, e1, e2); :}
-  | expr:e1 KW_MATCH_ELEMENT_EQ expr:e2
-  {: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_EQ, e1, e2); :}
-  | expr:e1 KW_MATCH_ELEMENT_LT expr:e2
-  {: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_LT, e1, e2); :}
-  | expr:e1 KW_MATCH_ELEMENT_GT expr:e2
-  {: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_GT, e1, e2); :}
-  | expr:e1 KW_MATCH_ELEMENT_LE expr:e2
-  {: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_LE, e1, e2); :}
-  | expr:e1 KW_MATCH_ELEMENT_GE expr:e2
-  {: RESULT = new MatchPredicate(MatchPredicate.Operator.MATCH_ELEMENT_GE, e1, e2); :}
   ;
 
 // Avoid a reduce/reduce conflict with compound_predicate by explicitly
@@ -8195,16 +8180,6 @@ keyword ::=
     {: RESULT = id; :}
     | KW_MATCH_PHRASE_EDGE:id
     {: RESULT = id; :}
-    | KW_MATCH_ELEMENT_EQ:id
-    {: RESULT = id; :}
-    | KW_MATCH_ELEMENT_LT:id
-    {: RESULT = id; :}
-    | KW_MATCH_ELEMENT_GT:id
-    {: RESULT = id; :}
-    | KW_MATCH_ELEMENT_LE:id
-    {: RESULT = id; :}
-    | KW_MATCH_ELEMENT_GE:id
-    {: RESULT = id; :}
     ;
 
 // Identifier that contain keyword