Skip to content

[Improvement](sink) optimization for parallel result sink #36667

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BiteTheDDDDt merged 2 commits into from
Jun 24, 2024
Merged

[Improvement](sink) optimization for parallel result sink #36667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
839751b
[Improvement](sink) optimization for parallel result sink (#36305)
BiteTheDDDDt Jun 19, 2024
95fa8a3
reset
BiteTheDDDDt Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions be/src/pipeline/exec/result_file_sink_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ Status ResultFileSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& i
if (p._is_top_sink) {
// create sender
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
state->fragment_instance_id(), p._buf_size, &_sender, state->execution_timeout()));
state->fragment_instance_id(), p._buf_size, &_sender, state->execution_timeout(),
state->batch_size()));
// create writer
_writer.reset(new (std::nothrow) vectorized::VFileResultWriter(
p._file_opts.get(), p._storage_type, state->fragment_instance_id(),
Expand Down Expand Up @@ -175,7 +176,7 @@ Status ResultFileSinkLocalState::close(RuntimeState* state, Status exec_status)
// close sender, this is normal path end
if (_sender) {
_sender->update_return_rows(_writer == nullptr ? 0 : _writer->get_written_rows());
RETURN_IF_ERROR(_sender->close(final_status));
RETURN_IF_ERROR(_sender->close(state->fragment_instance_id(), final_status));
}
state->exec_env()->result_mgr()->cancel_at_time(
time(nullptr) + config::result_buffer_cancelled_interval_time,
Expand Down
2 changes: 1 addition & 1 deletion be/src/pipeline/exec/result_file_sink_operator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ class ResultFileSinkOperatorX final : public DataSinkOperatorX<ResultFileSinkLoc

// Owned by the RuntimeState.
RowDescriptor _output_row_descriptor;
int _buf_size = 1024; // Allocated from _pool
int _buf_size = 4096; // Allocated from _pool
bool _is_top_sink = true;
std::string _header;
std::string _header_type;
Expand Down
13 changes: 7 additions & 6 deletions be/src/pipeline/exec/result_sink_operator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@ Status ResultSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& info)
_sender = _parent->cast<ResultSinkOperatorX>()._sender;
} else {
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
state->fragment_instance_id(), RESULT_SINK_BUFFER_SIZE, &_sender,
state->execution_timeout()));
fragment_instance_id, RESULT_SINK_BUFFER_SIZE, &_sender, state->execution_timeout(),
state->batch_size()));
}
_sender->set_dependency(_dependency->shared_from_this());
_sender->set_dependency(fragment_instance_id, _dependency->shared_from_this());
return Status::OK();
}

Expand Down Expand Up @@ -122,7 +122,8 @@ Status ResultSinkOperatorX::prepare(RuntimeState* state) {

if (state->query_options().enable_parallel_result_sink) {
RETURN_IF_ERROR(state->exec_env()->result_mgr()->create_sender(
state->query_id(), RESULT_SINK_BUFFER_SIZE, &_sender, state->execution_timeout()));
state->query_id(), RESULT_SINK_BUFFER_SIZE, &_sender, state->execution_timeout(),
state->batch_size()));
}
return Status::OK();
}
Expand All @@ -139,7 +140,7 @@ Status ResultSinkOperatorX::sink(RuntimeState* state, vectorized::Block* block,
if (_fetch_option.use_two_phase_fetch && block->rows() > 0) {
RETURN_IF_ERROR(_second_phase_fetch_data(state, block));
}
RETURN_IF_ERROR(local_state._writer->write(*block));
RETURN_IF_ERROR(local_state._writer->write(state, *block));
if (_fetch_option.use_two_phase_fetch) {
// Block structure may be changed by calling _second_phase_fetch_data().
// So we should clear block in case of unmatched columns
Expand Down Expand Up @@ -185,7 +186,7 @@ Status ResultSinkLocalState::close(RuntimeState* state, Status exec_status) {
if (_writer) {
_sender->update_return_rows(_writer->get_written_rows());
}
RETURN_IF_ERROR(_sender->close(final_status));
RETURN_IF_ERROR(_sender->close(state->fragment_instance_id(), final_status));
}
state->exec_env()->result_mgr()->cancel_at_time(
time(nullptr) + config::result_buffer_cancelled_interval_time,
Expand Down
2 changes: 1 addition & 1 deletion be/src/pipeline/exec/result_sink_operator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ struct ResultFileOptions {
}
};

constexpr int RESULT_SINK_BUFFER_SIZE = 4096;
constexpr int RESULT_SINK_BUFFER_SIZE = 4096 * 8;

class ResultSinkLocalState final : public PipelineXSinkLocalState<BasicSharedState> {
ENABLE_FACTORY_CREATOR(ResultSinkLocalState);
Expand Down
Loading