Skip to content

[FIX](array)fix array with inverted index #35926

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiaokang merged 16 commits into from
Jun 12, 2024
Merged

[FIX](array)fix array with inverted index #35926

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
fd4c513
fix for array inverted index writer with large dataset witch will mak…
Apr 24, 2024
b017368
delete match_element_xx and add fe config for create inverted index o…
May 13, 2024
cd38486
delete not defined
Jun 5, 2024
fed9ea4
update inverted index format
amorynan Jun 5, 2024
ffcd674
update clucene
Jun 5, 2024
8885519
fix none functions
Jun 5, 2024
b170d7e
fix code
Jun 5, 2024
6bf4496
format be code
Jun 5, 2024
7d1cd50
delete hard code limit
Jun 6, 2024
b1ef1fd
fix fe code
Jun 6, 2024
ed66f98
add import
Jun 6, 2024
d959732
format fe
Jun 6, 2024
27cf515
delete useless config
Jun 6, 2024
3e59513
delete unused import
Jun 6, 2024
992c111
fix cases
Jun 6, 2024
f571822
fix merge_on_write set true for agg table
Jun 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion be/src/clucene
Open in desktop
Submodule clucene updated 4 files
+7 −0 src/core/CLucene/document/Document.cpp
+6 −1 src/core/CLucene/document/Document.h
+27 −0 src/core/CLucene/index/SDocumentWriter.cpp
+2 −0 src/core/CLucene/index/SDocumentWriter.h
10 changes: 0 additions & 10 deletions be/src/exec/olap_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,16 +310,6 @@ class ColumnValueRange {
condition.__set_condition_op("match_phrase_prefix");
} else if (value.first == MatchType::MATCH_REGEXP) {
condition.__set_condition_op("match_regexp");
} else if (value.first == MatchType::MATCH_ELEMENT_EQ) {
condition.__set_condition_op("match_element_eq");
} else if (value.first == MatchType::MATCH_ELEMENT_LT) {
condition.__set_condition_op("match_element_lt");
} else if (value.first == MatchType::MATCH_ELEMENT_GT) {
condition.__set_condition_op("match_element_gt");
} else if (value.first == MatchType::MATCH_ELEMENT_LE) {
condition.__set_condition_op("match_element_le");
} else if (value.first == MatchType::MATCH_ELEMENT_GE) {
condition.__set_condition_op("match_element_ge");
}
condition.condition_values.push_back(
cast_to_string<primitive_type, CppType>(value.second, _scale));
Expand Down
41 changes: 2 additions & 39 deletions be/src/exec/olap_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,11 +169,6 @@ enum class MatchType {
MATCH_ANY = 0,
MATCH_ALL = 1,
MATCH_PHRASE = 2,
MATCH_ELEMENT_EQ = 3,
MATCH_ELEMENT_LT = 4,
MATCH_ELEMENT_GT = 5,
MATCH_ELEMENT_LE = 6,
MATCH_ELEMENT_GE = 7,
MATCH_PHRASE_PREFIX = 8,
MATCH_REGEXP = 9,
};
Expand All @@ -195,21 +190,6 @@ inline MatchType to_match_type(TExprOpcode::type type) {
case TExprOpcode::type::MATCH_REGEXP:
return MatchType::MATCH_REGEXP;
break;
case TExprOpcode::type::MATCH_ELEMENT_EQ:
return MatchType::MATCH_ELEMENT_EQ;
break;
case TExprOpcode::type::MATCH_ELEMENT_LT:
return MatchType::MATCH_ELEMENT_LT;
break;
case TExprOpcode::type::MATCH_ELEMENT_GT:
return MatchType::MATCH_ELEMENT_GT;
break;
case TExprOpcode::type::MATCH_ELEMENT_LE:
return MatchType::MATCH_ELEMENT_LE;
break;
case TExprOpcode::type::MATCH_ELEMENT_GE:
return MatchType::MATCH_ELEMENT_GE;
break;
default:
VLOG_CRITICAL << "TExprOpcode: " << type;
DCHECK(false);
Expand All @@ -228,16 +208,6 @@ inline MatchType to_match_type(const std::string& condition_op) {
return MatchType::MATCH_PHRASE_PREFIX;
} else if (condition_op.compare("match_regexp") == 0) {
return MatchType::MATCH_REGEXP;
} else if (condition_op.compare("match_element_eq") == 0) {
return MatchType::MATCH_ELEMENT_EQ;
} else if (condition_op.compare("match_element_lt") == 0) {
return MatchType::MATCH_ELEMENT_LT;
} else if (condition_op.compare("match_element_gt") == 0) {
return MatchType::MATCH_ELEMENT_GT;
} else if (condition_op.compare("match_element_le") == 0) {
return MatchType::MATCH_ELEMENT_LE;
} else if (condition_op.compare("match_element_ge") == 0) {
return MatchType::MATCH_ELEMENT_GE;
}
return MatchType::UNKNOWN;
}
Expand All @@ -246,12 +216,7 @@ inline bool is_match_condition(const std::string& op) {
if (0 == strcasecmp(op.c_str(), "match_any") || 0 == strcasecmp(op.c_str(), "match_all") ||
0 == strcasecmp(op.c_str(), "match_phrase") ||
0 == strcasecmp(op.c_str(), "match_phrase_prefix") ||
0 == strcasecmp(op.c_str(), "match_regexp") ||
0 == strcasecmp(op.c_str(), "match_element_eq") ||
0 == strcasecmp(op.c_str(), "match_element_lt") ||
0 == strcasecmp(op.c_str(), "match_element_gt") ||
0 == strcasecmp(op.c_str(), "match_element_le") ||
0 == strcasecmp(op.c_str(), "match_element_ge")) {
0 == strcasecmp(op.c_str(), "match_regexp")) {
return true;
}
return false;
Expand All @@ -260,9 +225,7 @@ inline bool is_match_condition(const std::string& op) {
inline bool is_match_operator(const TExprOpcode::type& op_type) {
return TExprOpcode::MATCH_ANY == op_type || TExprOpcode::MATCH_ALL == op_type ||
TExprOpcode::MATCH_PHRASE == op_type || TExprOpcode::MATCH_PHRASE_PREFIX == op_type ||
TExprOpcode::MATCH_REGEXP == op_type || TExprOpcode::MATCH_ELEMENT_EQ == op_type ||
TExprOpcode::MATCH_ELEMENT_LT == op_type || TExprOpcode::MATCH_ELEMENT_GT == op_type ||
TExprOpcode::MATCH_ELEMENT_LE == op_type || TExprOpcode::MATCH_ELEMENT_GE == op_type;
TExprOpcode::MATCH_REGEXP == op_type;
}

} // namespace doris
17 changes: 1 addition & 16 deletions be/src/olap/match_predicate.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,21 +101,6 @@ InvertedIndexQueryType MatchPredicate::_to_inverted_index_query_type(MatchType m
case MatchType::MATCH_REGEXP:
ret = InvertedIndexQueryType::MATCH_REGEXP_QUERY;
break;
case MatchType::MATCH_ELEMENT_EQ:
ret = InvertedIndexQueryType::EQUAL_QUERY;
break;
case MatchType::MATCH_ELEMENT_LT:
ret = InvertedIndexQueryType::LESS_THAN_QUERY;
break;
case MatchType::MATCH_ELEMENT_GT:
ret = InvertedIndexQueryType::GREATER_THAN_QUERY;
break;
case MatchType::MATCH_ELEMENT_LE:
ret = InvertedIndexQueryType::LESS_EQUAL_QUERY;
break;
case MatchType::MATCH_ELEMENT_GE:
ret = InvertedIndexQueryType::GREATER_EQUAL_QUERY;
break;
default:
DCHECK(false);
}
Expand All @@ -132,4 +117,4 @@ bool MatchPredicate::_skip_evaluate(InvertedIndexIterator* iterator) const {
return false;
}

} // namespace doris
} // namespace doris
Loading