Skip to content

[enhancement](delete-pred) enable delete sub predicate v2 for compaction #35859

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Jun 5, 2024
Merged

[enhancement](delete-pred) enable delete sub predicate v2 for compaction #35859

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 42 additions & 43 deletions be/src/olap/delete_handler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,9 @@
#include <gen_cpp/olap_file.pb.h>
#include <thrift/protocol/TDebugProtocol.h>

#include <algorithm>
#include <boost/regex.hpp>
#include <limits>
#include <sstream>
#include <string>
#include <utility>
#include <vector>

#include "common/config.h"
Expand All @@ -49,6 +46,47 @@ using ::google::protobuf::RepeatedPtrField;
namespace doris {
using namespace ErrorCode;

// construct sub condition from TCondition
std::string construct_sub_predicate(const TCondition& condition) {
string op = condition.condition_op;
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
string condition_str;
if ("IS" == op) {
// ATTN: tricky! Surround IS with spaces to make it "special"
condition_str = condition.column_name + " IS " + condition.condition_values[0];
} else { // multi-elements IN expr has been processed with InPredicatePB
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
condition_str = condition.column_name + op + "'" + condition.condition_values[0] + "'";
}
return condition_str;
}

// make operators from FE adaptive to BE
std::string trans_op(const std::string& opt) {
std::string op = string(opt);
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
if ("IS" != op) {
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
}
return op;
}

Status DeleteHandler::generate_delete_predicate(const TabletSchema& schema,
const std::vector<TCondition>& conditions,
DeletePredicatePB* del_pred) {
Expand Down Expand Up @@ -126,45 +164,6 @@ void DeleteHandler::convert_to_sub_pred_v2(DeletePredicatePB* delete_pred,
}
}

std::string DeleteHandler::construct_sub_predicate(const TCondition& condition) {
string op = condition.condition_op;
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
string condition_str;
if ("IS" == op) {
// ATTN: tricky! Surround IS with spaces to make it "special"
condition_str = condition.column_name + " IS " + condition.condition_values[0];
} else { // multi-elements IN expr has been processed with InPredicatePB
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
condition_str = condition.column_name + op + "'" + condition.condition_values[0] + "'";
}
return condition_str;
}

std::string DeleteHandler::trans_op(const std::string& opt) {
std::string op = string(opt);
if (op == "<") {
op += "<";
} else if (op == ">") {
op += ">";
}
if ("IS" != op) {
if (op == "*=") {
op = "=";
} else if (op == "!*=") {
op = "!=";
}
}
return op;
}

bool DeleteHandler::is_condition_value_valid(const TabletColumn& column,
const std::string& condition_op,
const string& value_str) {
Expand Down Expand Up @@ -395,7 +394,7 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema,
const auto& delete_condition = delete_pred->delete_predicate();
DeleteConditions temp;
temp.filter_version = delete_pred->version().first;
if (with_sub_pred_v2) {
if (with_sub_pred_v2 && !delete_condition.sub_predicates_v2().empty()) {
RETURN_IF_ERROR(_parse_column_pred(tablet_schema, delete_pred_related_schema,
delete_condition.sub_predicates_v2(), &temp));
} else {
Expand Down
14 changes: 3 additions & 11 deletions be/src/olap/delete_handler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,9 @@
#pragma once

#include <butil/macros.h>
Copy link
Contributor

@github-actions github-actions bot Jun 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: 'butil/macros.h' file not found [clang-diagnostic-error]

#include <butil/macros.h>
         ^

#include <stdint.h>

#include <memory>
#include <cstdint>
#include <string>
#include <unordered_map>
#include <vector>

#include "common/factory_creator.h"
#include "common/status.h"
Expand Down Expand Up @@ -88,12 +85,6 @@ class DeleteHandler {
const std::string& condition_op,
const std::string& value_str);

// construct sub condition from TCondition
static std::string construct_sub_predicate(const TCondition& condition);

// make operators from FE adaptive to BE
[[nodiscard]] static std::string trans_op(const string& op);

// extract 'column_name', 'op' and 'operands' to condition
static Status parse_condition(const DeleteSubPredicatePB& sub_cond, TCondition* condition);

Expand All @@ -107,7 +98,8 @@ class DeleteHandler {
// input:
// * schema: tablet's schema, the delete conditions and data rows are in this schema
// * version: maximum version
// * with_sub_pred_v2: whether to use delete sub predicate v2 (v2 is based on PB, v1 is based on condition string)
// * with_sub_pred_v2: whether to use delete sub predicate v2 (v2 is based on PB and use column uid to specify a column,
// v1 is based on condition string, and relies on regex for parse)
// return:
// * Status::Error<DELETE_INVALID_PARAMETERS>(): input parameters are not valid
// * Status::Error<MEM_ALLOC_FAILED>(): alloc memory failed
Expand Down
6 changes: 0 additions & 6 deletions be/src/olap/tablet_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -643,13 +643,7 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
((read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
config::enable_delete_when_cumu_compaction)) ||
read_params.reader_type == ReaderType::READER_CHECKSUM);
if (_filter_delete) {
// note(tsy): for compaction, keep delete sub pred v1 temporarily
return _delete_handler.init(_tablet_schema, read_params.delete_predicates,
read_params.version.second, false);
}
auto* runtime_state = read_params.runtime_state;
// note(tsy): for query, use session var to enable delete sub pred v2, for schema change, use v2 directly
bool enable_sub_pred_v2 =
runtime_state == nullptr ? true : runtime_state->enable_delete_sub_pred_v2();
return _delete_handler.init(_tablet_schema, read_params.delete_predicates,
Expand Down