diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h index 6a5f27bd32606c..24a35a3ba158c0 100644 --- a/be/src/olap/comparison_predicate.h +++ b/be/src/olap/comparison_predicate.h @@ -101,9 +101,11 @@ class ComparisonPredicateBase : public ColumnPredicate { std::shared_ptr roaring = std::make_shared(); - auto&& value = PrimitiveTypeConvertor::to_storage_field_type(_value); - RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name, &value, query_type, - num_rows, roaring)); + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR( + InvertedIndexQueryParamFactory::create_query_value(&_value, query_param)); + RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name, query_param->get_value(), + query_type, num_rows, roaring)); // mask out null_bitmap, since NULL cmp VALUE will produce NULL // and be treated as false in WHERE diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h index 4a1a10f898f0e6..5e10fdb62a864e 100644 --- a/be/src/olap/in_list_predicate.h +++ b/be/src/olap/in_list_predicate.h @@ -191,12 +191,15 @@ class InListPredicateBase : public ColumnPredicate { HybridSetBase::IteratorBase* iter = _values->begin(); while (iter->has_next()) { const void* ptr = iter->get_value(); - auto&& value = PrimitiveTypeConvertor::to_storage_field_type( - *reinterpret_cast(ptr)); + // auto&& value = PrimitiveTypeConvertor::to_storage_field_type( + // *reinterpret_cast(ptr)); + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR( + InvertedIndexQueryParamFactory::create_query_value(ptr, query_param)); InvertedIndexQueryType query_type = InvertedIndexQueryType::EQUAL_QUERY; std::shared_ptr index = std::make_shared(); - RETURN_IF_ERROR(iterator->read_from_inverted_index(column_name, &value, query_type, - num_rows, index)); + RETURN_IF_ERROR(iterator->read_from_inverted_index( + column_name, query_param->get_value(), query_type, num_rows, index)); indices |= *index; iter->next(); } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index f08dac8fb9bc28..522c7125db508d 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -73,6 +73,47 @@ namespace doris::segment_v2 { +template +Status InvertedIndexQueryParamFactory::create_query_value( + const void* value, std::unique_ptr& result_param) { + using CPP_TYPE = typename PrimitiveTypeTraits::CppType; + std::unique_ptr> param = + InvertedIndexQueryParam::create_unique(); + auto&& storage_val = PrimitiveTypeConvertor::to_storage_field_type( + *reinterpret_cast(value)); + param->set_value(&storage_val); + result_param = std::move(param); + return Status::OK(); +}; + +#define CREATE_QUERY_VALUE_TEMPLATE(PT) \ + template Status InvertedIndexQueryParamFactory::create_query_value( \ + const void* value, std::unique_ptr& result_param); + +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_BOOLEAN) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_TINYINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_SMALLINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_INT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_BIGINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_LARGEINT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_FLOAT) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DOUBLE) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_VARCHAR) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATE) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATEV2) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATETIME) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DATETIMEV2) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_CHAR) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMALV2) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL32) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL64) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL128I) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_DECIMAL256) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_HLL) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_STRING) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV4) +CREATE_QUERY_VALUE_TEMPLATE(PrimitiveType::TYPE_IPV6) + std::unique_ptr InvertedIndexReader::create_analyzer( InvertedIndexCtx* inverted_index_ctx) { std::unique_ptr analyzer; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index 64f78bd52d9afa..48450b974ac034 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -33,6 +33,7 @@ #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/rowset/segment_v2/inverted_index_query_type.h" #include "olap/tablet_schema.h" +#include "runtime/primitive_type.h" #include "util/once.h" #define FINALIZE_INPUT(x) \ @@ -279,6 +280,79 @@ class BkdIndexReader : public InvertedIndexReader { const KeyCoder* _value_key_coder {}; }; +/** + * @brief InvertedIndexQueryParamFactory is a factory class to create QueryValue object. + * we need a template function to make predict class like in_list_predict template class to use. + * also need a function with primitive type parameter to create inverted index query value. like some function expr: function_array_index + * Now we just mapping field value in query engine to storage field value + */ +class InvertedIndexQueryParamFactory { + ENABLE_FACTORY_CREATOR(InvertedIndexQueryParamFactory); + +public: + virtual ~InvertedIndexQueryParamFactory() = default; + + template + static Status create_query_value(const void* value, + std::unique_ptr& result_param); + + static Status create_query_value( + const PrimitiveType& primitiveType, const void* value, + std::unique_ptr& result_param) { + switch (primitiveType) { +#define M(TYPE) \ + case TYPE: { \ + return create_query_value(value, result_param); \ + } + M(PrimitiveType::TYPE_BOOLEAN) + M(PrimitiveType::TYPE_TINYINT) + M(PrimitiveType::TYPE_SMALLINT) + M(PrimitiveType::TYPE_INT) + M(PrimitiveType::TYPE_BIGINT) + M(PrimitiveType::TYPE_LARGEINT) + M(PrimitiveType::TYPE_FLOAT) + M(PrimitiveType::TYPE_DOUBLE) + M(PrimitiveType::TYPE_DECIMALV2) + M(PrimitiveType::TYPE_DECIMAL32) + M(PrimitiveType::TYPE_DECIMAL64) + M(PrimitiveType::TYPE_DECIMAL128I) + M(PrimitiveType::TYPE_DECIMAL256) + M(PrimitiveType::TYPE_DATE) + M(PrimitiveType::TYPE_DATETIME) + M(PrimitiveType::TYPE_CHAR) + M(PrimitiveType::TYPE_VARCHAR) + M(PrimitiveType::TYPE_STRING) +#undef M + default: + return Status::NotSupported("Unsupported primitive type {} for inverted index reader", + primitiveType); + } + }; + + virtual const void* get_value() const { + LOG_FATAL( + "Execution reached an undefined behavior code path in " + "InvertedIndexQueryParamFactory"); + __builtin_unreachable(); + }; +}; + +template +class InvertedIndexQueryParam : public InvertedIndexQueryParamFactory { + ENABLE_FACTORY_CREATOR(InvertedIndexQueryParam); + using storage_val = typename PrimitiveTypeTraits::StorageFieldType; + +public: + void set_value(const storage_val* value) { + _value = *reinterpret_cast(value); + } + + const void* get_value() const override { return &_value; } + +private: + storage_val _value; +}; + class InvertedIndexIterator { ENABLE_FACTORY_CREATOR(InvertedIndexIterator); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index e9956008f98179..784233918e67da 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -410,6 +410,8 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { RETURN_IF_ERROR(add_document()); _doc->clear(); _CLDELETE(ts); + } else { + RETURN_IF_ERROR(add_null_document()); } _rid++; } diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 506d7d0b2ad322..3adef84e7dda55 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -362,6 +362,12 @@ Status SegmentIterator::_lazy_init() { _segment->_tablet_schema->cluster_key_idxes().empty()) { RETURN_IF_ERROR(_get_row_ranges_by_keys()); } + // extract for index apply col id which is slot_ref + if (_enable_common_expr_pushdown && !_remaining_conjunct_roots.empty()) { + for (auto expr : _remaining_conjunct_roots) { + RETURN_IF_ERROR(_extract_common_expr_columns_for_index(expr)); + } + } RETURN_IF_ERROR(_get_row_ranges_by_column_conditions()); RETURN_IF_ERROR(_vec_init_lazy_materialization()); // Remove rows that have been marked deleted @@ -726,6 +732,20 @@ Status SegmentIterator::_extract_common_expr_columns(const vectorized::VExprSPtr return Status::OK(); } +Status SegmentIterator::_extract_common_expr_columns_for_index(const vectorized::VExprSPtr& expr) { + auto& children = expr->children(); + for (int i = 0; i < children.size(); ++i) { + RETURN_IF_ERROR(_extract_common_expr_columns_for_index(children[i])); + } + + auto node_type = expr->node_type(); + if (node_type == TExprNodeType::SLOT_REF) { + auto slot_expr = std::dynamic_pointer_cast(expr); + _common_expr_columns_for_index.insert(slot_expr->column_id()); + } + return Status::OK(); +} + Status SegmentIterator::_execute_predicates_except_leafnode_of_andnode( const vectorized::VExprSPtr& expr) { if (expr == nullptr) { @@ -815,6 +835,17 @@ bool SegmentIterator::_can_filter_by_preds_except_leafnode_of_andnode() { return true; } +bool SegmentIterator::_check_apply_by_inverted_index(ColumnId col_id) { + if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) { + return false; + } + if (_inverted_index_iterators[col_id] == nullptr) { + //this column without inverted index + return false; + } + return true; +} + bool SegmentIterator::_check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound) { if (_opts.runtime_state && !_opts.runtime_state->query_options().enable_inverted_index_query) { return false; @@ -1210,6 +1241,46 @@ Status SegmentIterator::_apply_inverted_index() { } } + // add a switch for inverted index filter + if (_opts.runtime_state && + _opts.runtime_state->enable_common_expr_pushdown_for_inverted_index()) { + // support expr to evaluate inverted index + std::unordered_map> + iter_map; + for (auto col_id : _common_expr_columns_for_index) { + auto tablet_col_id = _schema->column_id(col_id); + if (_check_apply_by_inverted_index(tablet_col_id)) { + iter_map[col_id] = std::make_pair(_storage_name_and_type[tablet_col_id], + _inverted_index_iterators[tablet_col_id].get()); + } + } + for (auto expr_ctx : _common_expr_ctxs_push_down) { + // _inverted_index_iterators has all column ids which has inverted index + // _common_expr_columns has all column ids from _common_expr_ctxs_push_down + // if current bitmap is already empty just return + if (_row_bitmap.isEmpty()) { + break; + } + std::shared_ptr result_bitmap = std::make_shared(); + if (Status st = + expr_ctx->eval_inverted_index(iter_map, num_rows(), result_bitmap.get()); + !st.ok()) { + if (_downgrade_without_index(st) || st.code() == ErrorCode::NOT_IMPLEMENTED_ERROR) { + continue; + } else { + // other code is not to be handled, we should just break + LOG(WARNING) << "failed to evaluate inverted index for expr_ctx: " + << expr_ctx->root()->debug_string() + << ", error msg: " << st.to_string(); + return st; + } + } else { + // every single result of expr_ctx must be `and` collection relationship + _row_bitmap &= *result_bitmap; + } + } + } + _col_predicates = std::move(remaining_predicates); _opts.stats->rows_inverted_index_filtered += (input_rows - _row_bitmap.cardinality()); return Status::OK(); diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index 84c10f3b8b2825..c01141509b553e 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -274,6 +274,8 @@ class SegmentIterator : public RowwiseIterator { bool _can_evaluated_by_vectorized(ColumnPredicate* predicate); [[nodiscard]] Status _extract_common_expr_columns(const vectorized::VExprSPtr& expr); + // same with _extract_common_expr_columns, but only extract columns that can be used for index + [[nodiscard]] Status _extract_common_expr_columns_for_index(const vectorized::VExprSPtr& expr); [[nodiscard]] Status _execute_common_expr(uint16_t* sel_rowid_idx, uint16_t& selected_size, vectorized::Block* block); uint16_t _evaluate_common_expr_filter(uint16_t* sel_rowid_idx, uint16_t selected_size, @@ -284,6 +286,7 @@ class SegmentIterator : public RowwiseIterator { void _convert_dict_code_for_predicate_if_necessary_impl(ColumnPredicate* predicate); + bool _check_apply_by_inverted_index(ColumnId col_id); bool _check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound = false); std::string _gen_predicate_result_sign(ColumnPredicate* predicate); @@ -409,6 +412,7 @@ class SegmentIterator : public RowwiseIterator { // columns to read after predicate evaluation and remaining expr execute std::vector _non_predicate_columns; std::set _common_expr_columns; + std::set _common_expr_columns_for_index; // remember the rowids we've read for the current row block. // could be a local variable of next_batch(), kept here to reuse vector memory std::vector _block_rowids; diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index 025d9517c28ef4..bbe1dbfb1ab653 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -178,6 +178,12 @@ class RuntimeState { _query_options.enable_common_expr_pushdown; } + bool enable_common_expr_pushdown_for_inverted_index() const { + return enable_common_expr_pushdown() && + _query_options.__isset.enable_common_expr_pushdown_for_inverted_index && + _query_options.enable_common_expr_pushdown_for_inverted_index; + }; + bool enable_faster_float_convert() const { return _query_options.__isset.faster_float_convert && _query_options.faster_float_convert; } diff --git a/be/src/vec/exprs/vcompound_pred.h b/be/src/vec/exprs/vcompound_pred.h index 2ede99cae63c77..4ff628548692d3 100644 --- a/be/src/vec/exprs/vcompound_pred.h +++ b/be/src/vec/exprs/vcompound_pred.h @@ -53,6 +53,75 @@ class VCompoundPred : public VectorizedFnCall { const std::string& expr_name() const override { return _expr_name; } + // 1. when meet 'or' conjunct: a or b, if b can apply index, return all rows, so b should not be extracted + // 2. when meet 'and' conjunct, function with column b can not apply inverted index + // eg. a and hash(b)=1, if b can apply index, but hash(b)=1 is not for index, so b should not be extracted + // but a and array_contains(b, 1), b can be applied inverted index, which b can be extracted + Status eval_inverted_index( + VExprContext* context, + const std::unordered_map>& + colid_to_inverted_index_iter, + uint32_t num_rows, roaring::Roaring* bitmap) const override { + std::shared_ptr res = std::make_shared(); + if (_op == TExprOpcode::COMPOUND_OR) { + for (auto child : _children) { + std::shared_ptr child_roaring = + std::make_shared(); + Status st = child->eval_inverted_index(context, colid_to_inverted_index_iter, + num_rows, child_roaring.get()); + if (!st.ok()) { + bitmap->addRange(0, num_rows); + return st; + } + if (child_roaring->cardinality() == 0) { + // means inverted index filter do not reduce any rows + // the left expr no need to be extracted by inverted index, + // and cur roaring is all rows which means this inverted index is not useful, + // do not need to calculate with res bitmap + bitmap->addRange(0, num_rows); + return Status::OK(); + } + *res |= *child_roaring; + } + *bitmap = *res; + } else if (_op == TExprOpcode::COMPOUND_AND) { + for (int i = 0; i < _children.size(); ++i) { + std::shared_ptr child_roaring = + std::make_shared(); + Status st = _children[0]->eval_inverted_index(context, colid_to_inverted_index_iter, + num_rows, child_roaring.get()); + if (!st.ok()) { + continue; + } + if (i == 0) { + *res = *child_roaring; + } else { + *res &= *child_roaring; + } + if (res->isEmpty()) { + // the left expr no need to be extracted by inverted index, just return 0 rows + // res bitmap will be zero + return Status::OK(); + } + } + *bitmap = *res; + } else if (_op == TExprOpcode::COMPOUND_NOT) { + Status st = _children[0]->eval_inverted_index(context, colid_to_inverted_index_iter, + num_rows, res.get()); + if (!st.ok()) { + return st; + } + std::shared_ptr all_rows = std::make_shared(); + all_rows->addRange(0, num_rows); + *bitmap = *all_rows - *res; + } else { + return Status::NotSupported( + "Compound operator must be AND or OR or Not can execute with inverted index."); + } + return Status::OK(); + } + Status execute(VExprContext* context, Block* block, int* result_column_id) override { if (children().size() == 1 || !_all_child_is_compound_and_not_const()) { return VectorizedFnCall::execute(context, block, result_column_id); diff --git a/be/src/vec/exprs/vectorized_fn_call.cpp b/be/src/vec/exprs/vectorized_fn_call.cpp index b84499653ebf9b..732d99d0f0ec93 100644 --- a/be/src/vec/exprs/vectorized_fn_call.cpp +++ b/be/src/vec/exprs/vectorized_fn_call.cpp @@ -141,6 +141,31 @@ void VectorizedFnCall::close(VExprContext* context, FunctionContext::FunctionSta VExpr::close(context, scope); } +Status VectorizedFnCall::eval_inverted_index( + VExprContext* context, + const std::unordered_map>& + colid_to_inverted_index_iter, + uint32_t num_rows, roaring::Roaring* bitmap) const { + DCHECK_GE(get_num_children(), 1); + if (get_child(0)->is_slot_ref()) { + auto* column_slot_ref = assert_cast(get_child(0).get()); + if (auto iter = colid_to_inverted_index_iter.find(column_slot_ref->column_id()); + iter != colid_to_inverted_index_iter.end()) { + const auto& pair = iter->second; + return _function->eval_inverted_index(context->fn_context(_fn_context_index), + pair.first, pair.second, num_rows, bitmap); + } else { + return Status::NotSupported("column id {} not found in colid_to_inverted_index_iter", + column_slot_ref->column_id()); + } + } else { + return Status::NotSupported("we can only eval inverted index for slot ref expr, but got ", + get_child(0)->expr_name()); + } + return Status::OK(); +} + Status VectorizedFnCall::_do_execute(doris::vectorized::VExprContext* context, doris::vectorized::Block* block, int* result_column_id, std::vector& args) { diff --git a/be/src/vec/exprs/vectorized_fn_call.h b/be/src/vec/exprs/vectorized_fn_call.h index 24cab0c94ba0d3..a3ce85bb588f9a 100644 --- a/be/src/vec/exprs/vectorized_fn_call.h +++ b/be/src/vec/exprs/vectorized_fn_call.h @@ -27,6 +27,7 @@ #include "udf/udf.h" #include "vec/core/column_numbers.h" #include "vec/exprs/vexpr.h" +#include "vec/exprs/vslot_ref.h" #include "vec/functions/function.h" namespace doris { @@ -50,6 +51,12 @@ class VectorizedFnCall : public VExpr { Status execute_runtime_fitler(doris::vectorized::VExprContext* context, doris::vectorized::Block* block, int* result_column_id, std::vector& args) override; + Status eval_inverted_index( + VExprContext* context, + const std::unordered_map>& + colid_to_inverted_index_iter, + uint32_t num_rows, roaring::Roaring* bitmap) const override; Status prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) override; Status open(RuntimeState* state, VExprContext* context, FunctionContext::FunctionStateScope scope) override; diff --git a/be/src/vec/exprs/vexpr.h b/be/src/vec/exprs/vexpr.h index 8ad62100e064f3..10467e579622b8 100644 --- a/be/src/vec/exprs/vexpr.h +++ b/be/src/vec/exprs/vexpr.h @@ -30,6 +30,7 @@ #include #include "common/status.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" #include "runtime/define_primitive_type.h" #include "runtime/large_int_value.h" #include "runtime/types.h" @@ -115,6 +116,16 @@ class VExpr { virtual Status execute(VExprContext* context, Block* block, int* result_column_id) = 0; + // execute current expr with inverted index to filter block. Given a roaringbitmap of match rows + virtual Status eval_inverted_index( + VExprContext* context, + const std::unordered_map>& + colid_to_inverted_index_iter, + uint32_t num_rows, roaring::Roaring* bitmap) const { + return Status::NotSupported("Not supported execute_with_inverted_index"); + } + // Only the 4th parameter is used in the runtime filter. In and MinMax need overwrite the // interface virtual Status execute_runtime_fitler(VExprContext* context, Block* block, diff --git a/be/src/vec/exprs/vexpr_context.cpp b/be/src/vec/exprs/vexpr_context.cpp index cdbf22a7209c05..c2a45180aac0af 100644 --- a/be/src/vec/exprs/vexpr_context.cpp +++ b/be/src/vec/exprs/vexpr_context.cpp @@ -119,6 +119,13 @@ int VExprContext::register_function_context(RuntimeState* state, const TypeDescr _fn_contexts.back()->set_check_overflow_for_decimal(state->check_overflow_for_decimal()); return _fn_contexts.size() - 1; } +Status VExprContext::eval_inverted_index( + const std::unordered_map>& + colid_to_inverted_index_iter, + uint32_t num_rows, roaring::Roaring* bitmap) { + return _root->eval_inverted_index(this, colid_to_inverted_index_iter, num_rows, bitmap); +} Status VExprContext::filter_block(VExprContext* vexpr_ctx, Block* block, int column_to_keep) { if (vexpr_ctx == nullptr || block->rows() == 0) { diff --git a/be/src/vec/exprs/vexpr_context.h b/be/src/vec/exprs/vexpr_context.h index 423e1aac12a540..fc4862ef6c1ea6 100644 --- a/be/src/vec/exprs/vexpr_context.h +++ b/be/src/vec/exprs/vexpr_context.h @@ -25,6 +25,7 @@ #include "common/factory_creator.h" #include "common/status.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" #include "runtime/types.h" #include "udf/udf.h" #include "vec/core/block.h" @@ -69,6 +70,21 @@ class VExprContext { return _fn_contexts[i].get(); } + // execute expr with inverted index which column a, b has inverted indexes + // but some situation although column b has indexes, but apply index is not useful, we should + // skip this expr, just do not apply index anymore. + /** + * @param colid_to_inverted_index_iter contains all column id to inverted index iterator mapping from segmentIterator + * @param num_rows number of rows in one segment. + * @param bitmap roaring bitmap to store the result. 0 is present filed by index. + * @return status not ok means execute failed. + */ + [[nodiscard]] Status eval_inverted_index( + const std::unordered_map>& + colid_to_inverted_index_iter, + uint32_t num_rows, roaring::Roaring* bitmap); + [[nodiscard]] static Status filter_block(VExprContext* vexpr_ctx, Block* block, int column_to_keep); diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h index d4a8fa32962fab..5221d4f51f3ed5 100644 --- a/be/src/vec/functions/array/function_array_index.h +++ b/be/src/vec/functions/array/function_array_index.h @@ -25,6 +25,10 @@ #include #include "common/status.h" +#include "olap/column_predicate.h" +#include "olap/rowset/segment_v2/inverted_index_query_type.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" +#include "runtime/primitive_type.h" #include "vec/columns/column.h" #include "vec/columns/column_array.h" #include "vec/columns/column_nullable.h" @@ -70,6 +74,11 @@ struct ArrayCountEqual { static constexpr void apply(ResultType& current, size_t j) noexcept { ++current; } }; +struct ParamValue { + PrimitiveType type; + Field value; +}; + template class FunctionArrayIndex : public IFunction { public: @@ -87,6 +96,57 @@ class FunctionArrayIndex : public IFunction { bool use_default_implementation_for_nulls() const override { return false; } + Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override { + if (scope == FunctionContext::THREAD_LOCAL) { + return Status::OK(); + } + + DCHECK(context->get_num_args() >= 1); + DCHECK(context->get_arg_type(0)->is_array_type()); + // now we only support same + std::shared_ptr state = std::make_shared(); + Field field; + if (context->get_constant_col(1)) { + context->get_constant_col(1)->column_ptr->get(0, field); + state->value = field; + state->type = context->get_arg_type(1)->type; + context->set_function_state(scope, state); + } + return Status::OK(); + } + + /** + * eval inverted index. we can filter array rows with inverted index iter + */ + Status eval_inverted_index(FunctionContext* context, + const vectorized::NameAndTypePair& data_type_with_name, + segment_v2::InvertedIndexIterator* iter, uint32_t num_rows, + roaring::Roaring* bitmap) const override { + std::shared_ptr roaring = std::make_shared(); + auto* param_value = reinterpret_cast( + context->get_function_state(FunctionContext::FRAGMENT_LOCAL)); + + std::unique_ptr query_param = nullptr; + RETURN_IF_ERROR(InvertedIndexQueryParamFactory::create_query_value( + param_value->type, ¶m_value->value, query_param)); + RETURN_IF_ERROR(iter->read_from_inverted_index( + data_type_with_name.first, query_param->get_value(), + segment_v2::InvertedIndexQueryType::MATCH_ANY_QUERY, num_rows, roaring)); + + // mask out null_bitmap, since NULL cmp VALUE will produce NULL + // and be treated as false in WHERE + // keep it after query, since query will try to read null_bitmap and put it to cache + segment_v2::InvertedIndexQueryCacheHandle null_bitmap_cache_handle; + RETURN_IF_ERROR(iter->read_null_bitmap(&null_bitmap_cache_handle)); + std::shared_ptr null_bitmap = null_bitmap_cache_handle.get_bitmap(); + if (null_bitmap) { + *bitmap -= *null_bitmap; + } + + *bitmap = *roaring; + return Status::OK(); + } + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { if constexpr (OldVersion) { return make_nullable(std::make_shared>()); diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h index 30a849b4724211..c2eb31d6a4766d 100644 --- a/be/src/vec/functions/function.h +++ b/be/src/vec/functions/function.h @@ -31,6 +31,7 @@ #include "common/exception.h" #include "common/status.h" +#include "olap/rowset/segment_v2/inverted_index_reader.h" #include "udf/udf.h" #include "vec/core/block.h" #include "vec/core/column_numbers.h" @@ -179,6 +180,14 @@ class IFunctionBase { ->execute(context, block, arguments, result, input_rows_count, dry_run); } + virtual Status eval_inverted_index(FunctionContext* context, + const vectorized::NameAndTypePair& data_type_with_name, + segment_v2::InvertedIndexIterator* iter, uint32_t num_rows, + roaring::Roaring* bitmap) const { + return Status::NotSupported("eval_inverted_index is not supported in function: ", + get_name()); + } + /// Do cleaning work when function is finished, i.e., release state variables in the /// `FunctionContext` which are registered in `prepare` phase. virtual Status close(FunctionContext* context, FunctionContext::FunctionStateScope scope) { @@ -395,6 +404,15 @@ class IFunction : public std::enable_shared_from_this, return Status::OK(); } + // here are lots of function not extends eval_inverted_index. + Status eval_inverted_index(FunctionContext* context, + const vectorized::NameAndTypePair& data_type_with_name, + segment_v2::InvertedIndexIterator* iter, uint32_t num_rows, + roaring::Roaring* bitmap) const override { + return Status::NotSupported("eval_inverted_index is not supported in function: ", + get_name()); + } + [[noreturn]] const DataTypes& get_argument_types() const final { LOG(FATAL) << "get_argument_types is not implemented for IFunction"; __builtin_unreachable(); @@ -428,6 +446,14 @@ class DefaultExecutable final : public PreparedFunctionImpl { size_t result, size_t input_rows_count) const final { return function->execute_impl(context, block, arguments, result, input_rows_count); } + + Status eval_inverted_index(FunctionContext* context, + const vectorized::NameAndTypePair& data_type_with_name, + segment_v2::InvertedIndexIterator* iter, uint32_t num_rows, + roaring::Roaring* bitmap) const { + return function->eval_inverted_index(context, data_type_with_name, iter, num_rows, bitmap); + } + Status execute_impl_dry_run(FunctionContext* context, Block& block, const ColumnNumbers& arguments, size_t result, size_t input_rows_count) const final { @@ -491,6 +517,13 @@ class DefaultFunction final : public IFunctionBase { function_name == "gt" || function_name == "le" || function_name == "ge"; } + Status eval_inverted_index(FunctionContext* context, + const vectorized::NameAndTypePair& data_type_with_name, + segment_v2::InvertedIndexIterator* iter, uint32_t num_rows, + roaring::Roaring* bitmap) const override { + return function->eval_inverted_index(context, data_type_with_name, iter, num_rows, bitmap); + } + IFunctionBase::Monotonicity get_monotonicity_for_range(const IDataType& type, const Field& left, const Field& right) const override { return function->get_monotonicity_for_range(type, left, right); diff --git a/be/test/util/roaring_bitmap_test.cpp b/be/test/util/roaring_bitmap_test.cpp new file mode 100644 index 00000000000000..157e02ca2f5d45 --- /dev/null +++ b/be/test/util/roaring_bitmap_test.cpp @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "gtest/gtest_pred_impl.h" +#include "roaring/roaring.hh" + +namespace doris { + +TEST(RaringBitmapTest, IsAllOne) { + std::shared_ptr bitmap = std::make_shared(); + bitmap->addRange(0, 1024); + EXPECT_TRUE(bitmap->contains(1)); + EXPECT_FALSE(bitmap->contains(1025)); + EXPECT_EQ(bitmap->cardinality(), 1024); + + std::shared_ptr bitmap2 = std::make_shared(); + bitmap2->addRange(26, 31); + // and + *bitmap &= *bitmap2; + EXPECT_TRUE(bitmap->contains(26)); + EXPECT_FALSE(bitmap->contains(25)); + EXPECT_EQ(bitmap->cardinality(), 5); + + // or + std::shared_ptr bitmap3 = std::make_shared(); + bitmap3->addRange(0, 1024); + *bitmap |= *bitmap3; + EXPECT_TRUE(bitmap->contains(1)); + EXPECT_TRUE(bitmap->contains(31)); + EXPECT_FALSE(bitmap->contains(1025)); + + // not + std::shared_ptr bitmap4 = std::make_shared(); + bitmap4->addRange(32, 2048); + *bitmap -= *bitmap4; + EXPECT_EQ(0, bitmap->minimum()); + EXPECT_EQ(bitmap->maximum(), 31); + EXPECT_EQ(bitmap->cardinality(), 32); +} + +} // namespace doris \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 433997035c4f74..78358786c055cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -385,6 +385,9 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_INVERTED_INDEX_QUERY = "enable_inverted_index_query"; + public static final String ENABLE_COMMON_EXPR_PUSHDOWN_FOR_INVERTED_INDEX + = "enable_common_expr_pushdown_for_inverted_index"; + public static final String ENABLE_PUSHDOWN_COUNT_ON_INDEX = "enable_count_on_index_pushdown"; public static final String GROUP_BY_AND_HAVING_USE_ALIAS_FIRST = "group_by_and_having_use_alias_first"; @@ -1368,6 +1371,11 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { "是否启用inverted index query。", "Set whether to use inverted index query."}) public boolean enableInvertedIndexQuery = true; + // Whether enable query expr with inverted index. + @VariableMgr.VarAttr(name = ENABLE_COMMON_EXPR_PUSHDOWN_FOR_INVERTED_INDEX, needForward = true, description = { + "是否启用表达式上使用 inverted index。", "Set whether to use inverted index query for expr."}) + public boolean enableCommonExpPushDownForInvertedIndex = false; + // Whether enable pushdown count agg to scan node when using inverted index match. @VariableMgr.VarAttr(name = ENABLE_PUSHDOWN_COUNT_ON_INDEX, needForward = true, description = { "是否启用count_on_index pushdown。", "Set whether to pushdown count_on_index."}) @@ -3055,6 +3063,16 @@ public void setEnableInvertedIndexQuery(boolean enableInvertedIndexQuery) { this.enableInvertedIndexQuery = enableInvertedIndexQuery; } + + public boolean isEnableCommonExprPushdownForInvertedIndex() { + return enableCommonExpPushDownForInvertedIndex; + } + + + public void setEnableCommonExprPushdownForInvertedIndex(boolean enableCommonExpPushDownForInvertedIndex) { + this.enableCommonExpPushDownForInvertedIndex = enableCommonExpPushDownForInvertedIndex; + } + public boolean isEnablePushDownCountOnIndex() { return enablePushDownCountOnIndex; } @@ -3198,6 +3216,7 @@ public TQueryOptions toThrift() { tResult.setFileCacheBasePath(fileCacheBasePath); tResult.setEnableInvertedIndexQuery(enableInvertedIndexQuery); + tResult.setEnableCommonExprPushdownForInvertedIndex(enableCommonExpPushDownForInvertedIndex); if (dryRunQuery) { tResult.setDryRunQuery(true); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index 24d56ef8df01fe..63396fc5aa7001 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -289,6 +289,9 @@ struct TQueryOptions { // max rows of each sub-queue in DataQueue. 106: optional i64 data_queue_max_blocks = 0; + // expr pushdown for index filter rows + 107: optional bool enable_common_expr_pushdown_for_inverted_index = false; + // For cloud, to control if the content would be written into file cache 1000: optional bool disable_file_cache = false } diff --git a/regression-test/data/inverted_index_p0/test_array_contains_with_inverted_index.out b/regression-test/data/inverted_index_p0/test_array_contains_with_inverted_index.out new file mode 100644 index 00000000000000..76a72d8c595b01 --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_array_contains_with_inverted_index.out @@ -0,0 +1,82 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +16 + +-- !sql -- +2019-01-01 a648a447b8f71522f11632eba4b4adde ["p", "q", "r", "s", "t"] + +-- !sql -- + +-- !sql -- +2019-01-01 a648a447b8f71522f11632eba4b4adde ["p", "q", "r", "s", "t"] + +-- !sql -- +2017-01-01 021603e7dcfe65d44af0efd0e5aee154 ["n"] +2017-01-01 48a33ec3453a28bce84b8f96fe161956 ["m"] +2017-01-01 6afef581285b6608bf80d5a4e46cf839 ["a", "b", "c"] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a3 \N +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a4 \N +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a5 [] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a6 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a7 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a8 [] +2017-01-01 9fcb57ae675f0af4d613d9e6c0e8a2a2 ["o"] +2017-01-01 d93d942d985a8fb7547c72dada8d332d ["d", "e", "f", "g", "h", "i", "j", "k", "l"] +2019-01-01 a648a447b8f71522f11632eba4b4adde ["p", "q", "r", "s", "t"] + +-- !sql -- +2017-01-01 021603e7dcfe65d44af0efd0e5aee154 ["n"] +2017-01-01 48a33ec3453a28bce84b8f96fe161956 ["m"] +2017-01-01 6afef581285b6608bf80d5a4e46cf839 ["a", "b", "c"] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a5 [] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a6 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a7 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a8 [] +2017-01-01 9fcb57ae675f0af4d613d9e6c0e8a2a2 ["o"] +2017-01-01 d93d942d985a8fb7547c72dada8d332d ["d", "e", "f", "g", "h", "i", "j", "k", "l"] +2019-01-01 0974e7a82e30d1af83205e474fadd0a2 ["w"] +2019-01-01 26823b3995ee38bd145ddd910b2f6300 ["x"] +2019-01-01 a9fb5c985c90bf05f3bee5ca3ae95260 ["u", "v"] +2019-01-01 ee27ee1da291e46403c408e220bed6e1 ["y"] + +-- !sql -- +2017-01-01 021603e7dcfe65d44af0efd0e5aee154 ["n"] +2017-01-01 48a33ec3453a28bce84b8f96fe161956 ["m"] +2017-01-01 6afef581285b6608bf80d5a4e46cf839 ["a", "b", "c"] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a5 [] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a6 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a7 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a8 [] +2017-01-01 9fcb57ae675f0af4d613d9e6c0e8a2a2 ["o"] +2017-01-01 d93d942d985a8fb7547c72dada8d332d ["d", "e", "f", "g", "h", "i", "j", "k", "l"] + +-- !sql -- +2019-01-01 0974e7a82e30d1af83205e474fadd0a2 ["w"] +2019-01-01 26823b3995ee38bd145ddd910b2f6300 ["x"] +2019-01-01 a9fb5c985c90bf05f3bee5ca3ae95260 ["u", "v"] +2019-01-01 ee27ee1da291e46403c408e220bed6e1 ["y"] + +-- !sql -- +2017-01-01 021603e7dcfe65d44af0efd0e5aee154 ["n"] +2017-01-01 48a33ec3453a28bce84b8f96fe161956 ["m"] +2017-01-01 6afef581285b6608bf80d5a4e46cf839 ["a", "b", "c"] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a3 \N +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a4 \N +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a5 [] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a6 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a7 [null, null, null] +2017-01-01 8fcb57ae675f0af4d613d9e6c0e8a2a8 [] +2017-01-01 9fcb57ae675f0af4d613d9e6c0e8a2a2 ["o"] +2017-01-01 d93d942d985a8fb7547c72dada8d332d ["d", "e", "f", "g", "h", "i", "j", "k", "l"] +2019-01-01 0974e7a82e30d1af83205e474fadd0a2 ["w"] +2019-01-01 26823b3995ee38bd145ddd910b2f6300 ["x"] +2019-01-01 a9fb5c985c90bf05f3bee5ca3ae95260 ["u", "v"] +2019-01-01 ee27ee1da291e46403c408e220bed6e1 ["y"] + +-- !sql -- +2019-01-01 0974e7a82e30d1af83205e474fadd0a2 ["w"] +2019-01-01 26823b3995ee38bd145ddd910b2f6300 ["x"] +2019-01-01 a648a447b8f71522f11632eba4b4adde ["p", "q", "r", "s", "t"] +2019-01-01 a9fb5c985c90bf05f3bee5ca3ae95260 ["u", "v"] +2019-01-01 ee27ee1da291e46403c408e220bed6e1 ["y"] + diff --git a/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy new file mode 100644 index 00000000000000..0ea18c784d6340 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_array_contains_with_inverted_index.groovy @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_array_contains_with_inverted_index"){ + // prepare test table + def indexTblName = "tai" + + // If we use common expr pass to inverted index , we should set enable_common_expr_pushdown = true + sql """ set enable_common_expr_pushdown = true; """ + sql """ set enable_common_expr_pushdown_for_inverted_index = true; """ + sql """ set enable_pipeline_x_engine = true;""" + sql """ set enable_profile = true;""" + + sql "DROP TABLE IF EXISTS ${indexTblName}" + // create 1 replica table + sql """ + CREATE TABLE IF NOT EXISTS `${indexTblName}` ( + `apply_date` date NULL COMMENT '', + `id` varchar(60) NOT NULL COMMENT '', + `inventors` array NULL COMMENT '', + INDEX index_inverted_inventors(inventors) USING INVERTED COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`apply_date`, `id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "is_being_synced" = "false", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); + """ + + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '6afef581285b6608bf80d5a4e46cf839', '[\"a\", \"b\", \"c\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', 'd93d942d985a8fb7547c72dada8d332d', '[\"d\", \"e\", \"f\", \"g\", \"h\", \"i\", \"j\", \"k\", \"l\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '48a33ec3453a28bce84b8f96fe161956', '[\"m\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '021603e7dcfe65d44af0efd0e5aee154', '[\"n\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '9fcb57ae675f0af4d613d9e6c0e8a2a2', '[\"o\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a3'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a4', NULL); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a5', '[]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a6', '[null,null,null]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a7', [null,null,null]); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2017-01-01', '8fcb57ae675f0af4d613d9e6c0e8a2a8', []); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'a648a447b8f71522f11632eba4b4adde', '[\"p\", \"q\", \"r\", \"s\", \"t\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'a9fb5c985c90bf05f3bee5ca3ae95260', '[\"u\", \"v\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '0974e7a82e30d1af83205e474fadd0a2', '[\"w\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', '26823b3995ee38bd145ddd910b2f6300', '[\"x\"]'); """ + sql """ INSERT INTO `${indexTblName}`(`apply_date`, `id`, `inventors`) VALUES ('2019-01-01', 'ee27ee1da291e46403c408e220bed6e1', '[\"y\"]'); """ + + qt_sql """ select count() from ${indexTblName}""" + order_qt_sql """ select * from tai where array_contains(inventors, 's') order by id; """ + + order_qt_sql """ select * from tai where array_contains(inventors, 's') and apply_date = '2017-01-01' order by id; """ + order_qt_sql """ select * from tai where array_contains(inventors, 's') and apply_date = '2019-01-01' order by id; """ + order_qt_sql """ select * from tai where array_contains(inventors, 's') or apply_date = '2017-01-01' order by id; """ + order_qt_sql """ select * from tai where !array_contains(inventors, 's') order by id; """ + order_qt_sql """ select * from tai where !array_contains(inventors, 's') and apply_date = '2017-01-01' order by id; """ + order_qt_sql """ select * from tai where !array_contains(inventors, 's') and apply_date = '2019-01-01' order by id; """ + order_qt_sql """ select * from tai where !array_contains(inventors, 's') or apply_date = '2017-01-01' order by id; """ + order_qt_sql """ select * from tai where (array_contains(inventors, 's') and apply_date = '2017-01-01') or apply_date = '2019-01-01' order by id; """ +}