Skip to content

[coverage](test) Delete unused function to improve test coverage #25233

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gabriel39 merged 4 commits into from
Oct 11, 2023
Merged

[coverage](test) Delete unused function to improve test coverage #25233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
84ef398
update
Gabriel39 Oct 10, 2023
4b6d83c
[coverage](test) Delete unused function to improve test coverage
Gabriel39 Oct 10, 2023
f7062cc
update
Gabriel39 Oct 10, 2023
735865a
update
Gabriel39 Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 0 additions & 78 deletions be/src/runtime/datetime_value.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,6 @@

#pragma once

#include <re2/re2.h>
#include <stdint.h>

// IWYU pragma: no_include <bits/chrono.h>
#include <chrono>
#include <cstddef>
#include <iostream>

#include "cctz/civil_time.h"
#include "cctz/time_zone.h"
#include "udf/udf.h"
#include "util/hash_util.hpp"
#include "util/timezone_utils.h"
#include "vec/runtime/vdatetime_value.h"

namespace doris {

enum TimeUnit {
Expand All @@ -57,72 +42,9 @@ enum TimeUnit {
YEAR_MONTH
};

struct TimeInterval {
int64_t year;
int64_t month;
int64_t day;
int64_t hour;
int64_t minute;
int64_t second;
int64_t microsecond;
bool is_neg;

TimeInterval()
: year(0),
month(0),
day(0),
hour(0),
minute(0),
second(0),
microsecond(0),
is_neg(false) {}

TimeInterval(TimeUnit unit, int64_t count, bool is_neg_param)
: year(0),
month(0),
day(0),
hour(0),
minute(0),
second(0),
microsecond(0),
is_neg(is_neg_param) {
switch (unit) {
case YEAR:
year = count;
break;
case MONTH:
month = count;
break;
case WEEK:
day = 7 * count;
break;
case DAY:
day = count;
break;
case HOUR:
hour = count;
break;
case MINUTE:
minute = count;
break;
case SECOND:
second = count;
break;
case MICROSECOND:
microsecond = count;
break;
default:
break;
}
}
};

enum TimeType { TIME_TIME = 1, TIME_DATE = 2, TIME_DATETIME = 3 };

// 9999-99-99 99:99:99.999999; 26 + 1('\0')
const int MAX_DTVALUE_STR_LEN = 27;

constexpr size_t const_length(const char* str) {
return (str == nullptr || *str == 0) ? 0 : const_length(str + 1) + 1;
}
} // namespace doris
5 changes: 5 additions & 0 deletions be/src/runtime/runtime_state.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,4 +452,9 @@ std::shared_ptr<doris::pipeline::PipelineXSinkLocalStateBase> RuntimeState::get_
return _op_id_to_sink_local_state[id];
}

bool RuntimeState::enable_page_cache() const {
return !config::disable_storage_page_cache &&
(_query_options.__isset.enable_page_cache && _query_options.enable_page_cache);
}

} // end namespace doris
5 changes: 1 addition & 4 deletions be/src/runtime/runtime_state.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,10 +356,7 @@ class RuntimeState {
return _query_options.__isset.skip_delete_bitmap && _query_options.skip_delete_bitmap;
}

bool enable_page_cache() const {
return !config::disable_storage_page_cache &&
(_query_options.__isset.enable_page_cache && _query_options.enable_page_cache);
}
bool enable_page_cache() const;
Copy link
Contributor

@github-actions github-actions bot Oct 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: function 'enable_page_cache' should be marked [[nodiscard]] [modernize-use-nodiscard]

Suggested change
bool enable_page_cache() const;
[[nodiscard]] bool enable_page_cache() const;


int partitioned_hash_join_rows_threshold() const {
if (!_query_options.__isset.partitioned_hash_join_rows_threshold) {
Expand Down
102 changes: 0 additions & 102 deletions be/src/vec/common/schema_util.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,48 +154,6 @@ Status cast_column(const ColumnWithTypeAndName& arg, const DataTypePtr& type, Co
return Status::OK();
}

static void get_column_def(const vectorized::DataTypePtr& data_type, const std::string& name,
TColumnDef* column) {
if (!name.empty()) {
column->columnDesc.__set_columnName(name);
}
if (data_type->is_nullable()) {
const auto& real_type = static_cast<const DataTypeNullable&>(*data_type);
column->columnDesc.__set_isAllowNull(true);
get_column_def(real_type.get_nested_type(), "", column);
return;
}
column->columnDesc.__set_columnType(data_type->get_type_as_tprimitive_type());
if (data_type->get_type_id() == TypeIndex::Array) {
TColumnDef child;
column->columnDesc.__set_children({});
get_column_def(assert_cast<const DataTypeArray*>(data_type.get())->get_nested_type(), "",
&child);
column->columnDesc.columnLength =
TabletColumn::get_field_length_by_type(column->columnDesc.columnType, 0);
column->columnDesc.children.push_back(child.columnDesc);
return;
}
if (data_type->get_type_id() == TypeIndex::Tuple) {
// TODO
// auto tuple_type = assert_cast<const DataTypeTuple*>(data_type.get());
// DCHECK_EQ(tuple_type->get_elements().size(), tuple_type->get_element_names().size());
// for (size_t i = 0; i < tuple_type->get_elements().size(); ++i) {
// TColumnDef child;
// get_column_def(tuple_type->get_element(i), tuple_type->get_element_names()[i], &child);
// column->columnDesc.children.push_back(child.columnDesc);
// }
// return;
}
if (data_type->get_type_id() == TypeIndex::String) {
return;
}
if (WhichDataType(*data_type).is_simple()) {
column->columnDesc.__set_columnLength(data_type->get_size_of_value_in_memory());
return;
}
}

// send an empty add columns rpc, the rpc response will fill with base schema info
// maybe we could seperate this rpc from add columns rpc
Status send_fetch_full_base_schema_view_rpc(FullBaseSchemaView* schema_view) {
Expand Down Expand Up @@ -232,66 +190,6 @@ Status send_fetch_full_base_schema_view_rpc(FullBaseSchemaView* schema_view) {
static const std::regex COLUMN_NAME_REGEX(
"^[_a-zA-Z@0-9\\s<>/][.a-zA-Z0-9_+-/><?@#$%^&*\"\\s,:]{0,255}$");

// Do batch add columns schema change
// only the base table supported
Status send_add_columns_rpc(ColumnsWithTypeAndName column_type_names,
FullBaseSchemaView* schema_view) {
if (column_type_names.empty()) {
return Status::OK();
}
TAddColumnsRequest req;
TAddColumnsResult res;
TTabletInfo tablet_info;
req.__set_table_name(schema_view->table_name);
req.__set_db_name(schema_view->db_name);
req.__set_table_id(schema_view->table_id);
// TODO(lhy) more configurable
req.__set_allow_type_conflict(true);
req.__set_addColumns({});
// Deduplicate Column like `Level` and `level`
// TODO we will implement new version of dynamic column soon to handle this issue,
// also ignore column missmatch with regex
std::set<std::string> dedup;
for (const auto& column_type_name : column_type_names) {
if (dedup.contains(to_lower(column_type_name.name))) {
continue;
}
if (!std::regex_match(column_type_name.name, COLUMN_NAME_REGEX)) {
continue;
}
dedup.insert(to_lower(column_type_name.name));
TColumnDef col;
get_column_def(column_type_name.type, column_type_name.name, &col);
req.addColumns.push_back(col);
}
auto master_addr = ExecEnv::GetInstance()->master_info()->network_address;
Status rpc_st = ThriftRpcHelper::rpc<FrontendServiceClient>(
master_addr.hostname, master_addr.port,
[&req, &res](FrontendServiceConnection& client) { client->addColumns(res, req); },
config::txn_commit_rpc_timeout_ms);
if (!rpc_st.ok()) {
return Status::InternalError("Failed to do schema change, rpc error");
}
// TODO(lhy) handle more status code
if (res.status.status_code != TStatusCode::OK) {
LOG(WARNING) << "failed to do schema change, code:" << res.status.status_code
<< ", msg:" << res.status.error_msgs[0];
return Status::InvalidArgument(
fmt::format("Failed to do schema change, {}", res.status.error_msgs[0]));
}
size_t sz = res.allColumns.size();
if (sz < dedup.size()) {
return Status::InternalError(
fmt::format("Unexpected result columns {}, expected at least {}",
res.allColumns.size(), column_type_names.size()));
}
for (const auto& column : res.allColumns) {
schema_view->column_name_to_column[column.column_name] = column;
}
schema_view->schema_version = res.schema_version;
return Status::OK();
}

Status unfold_object(size_t dynamic_col_position, Block& block, bool cast_to_original_type,
RuntimeState* state) {
auto dynamic_col = block.get_by_position(dynamic_col_position).column->assume_mutable();
Expand Down
5 changes: 0 additions & 5 deletions be/src/vec/common/schema_util.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,13 +96,8 @@ struct FullBaseSchemaView {
int32_t table_id = 0;
std::string table_name;
std::string db_name;

bool empty() { return column_name_to_column.empty() && schema_version == -1; }
};

Status send_add_columns_rpc(ColumnsWithTypeAndName column_type_names,
FullBaseSchemaView* schema_view);

Status send_fetch_full_base_schema_view_rpc(FullBaseSchemaView* schema_view);

// For tracking local schema change during load procedure
Expand Down
13 changes: 0 additions & 13 deletions be/src/vec/core/materialize_block.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,6 @@

namespace doris::vectorized {

Block materialize_block(const Block& block) {
if (!block.columns()) return block;

Block res = block;
size_t columns = res.columns();
for (size_t i = 0; i < columns; ++i) {
auto& element = res.get_by_position(i);
element.column = element.column->convert_to_full_column_if_const();
}

return res;
}

void materialize_block_inplace(Block& block) {
for (size_t i = 0; i < block.columns(); ++i) {
block.replace_by_position_if_const(i);
Expand Down
1 change: 0 additions & 1 deletion be/src/vec/core/materialize_block.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ namespace doris::vectorized {

/** Converts columns-constants to full columns ("materializes" them).
*/
Block materialize_block(const Block& block);
void materialize_block_inplace(Block& block);

template <typename Iterator>
Expand Down
90 changes: 0 additions & 90 deletions be/src/vec/core/sort_block.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,26 +42,6 @@ ColumnsWithSortDescriptions get_columns_with_sort_description(const Block& block
return res;
}

struct PartialSortingLess {
const ColumnsWithSortDescriptions& columns;

explicit PartialSortingLess(const ColumnsWithSortDescriptions& columns_) : columns(columns_) {}

bool operator()(size_t a, size_t b) const {
for (ColumnsWithSortDescriptions::const_iterator it = columns.begin(); it != columns.end();
++it) {
int res = it->second.direction *
it->first->compare_at(a, b, *it->first, it->second.nulls_direction);
if (res < 0) {
return true;
} else if (res > 0) {
return false;
}
}
return false;
}
};

void sort_block(Block& src_block, Block& dest_block, const SortDescription& description,
UInt64 limit) {
if (!src_block.columns()) {
Expand Down Expand Up @@ -117,74 +97,4 @@ void sort_block(Block& src_block, Block& dest_block, const SortDescription& desc
}
}

void stable_get_permutation(const Block& block, const SortDescription& description,
IColumn::Permutation& out_permutation) {
if (!block.columns()) {
return;
}

size_t size = block.rows();
out_permutation.resize(size);
for (size_t i = 0; i < size; ++i) {
out_permutation[i] = i;
}

ColumnsWithSortDescriptions columns_with_sort_desc =
get_columns_with_sort_description(block, description);

std::stable_sort(out_permutation.begin(), out_permutation.end(),
PartialSortingLess(columns_with_sort_desc));
}

bool is_already_sorted(const Block& block, const SortDescription& description) {
if (!block.columns()) {
return true;
}

size_t rows = block.rows();

ColumnsWithSortDescriptions columns_with_sort_desc =
get_columns_with_sort_description(block, description);

PartialSortingLess less(columns_with_sort_desc);

/** If the rows are not too few, then let's make a quick attempt to verify that the block is not sorted.
* Constants - at random.
*/
static constexpr size_t num_rows_to_try = 10;
if (rows > num_rows_to_try * 5) {
for (size_t i = 1; i < num_rows_to_try; ++i) {
size_t prev_position = rows * (i - 1) / num_rows_to_try;
size_t curr_position = rows * i / num_rows_to_try;

if (less(curr_position, prev_position)) {
return false;
}
}
}

for (size_t i = 1; i < rows; ++i) {
if (less(i, i - 1)) {
return false;
}
}

return true;
}

void stable_sort_block(Block& block, const SortDescription& description) {
if (!block.columns()) {
return;
}

IColumn::Permutation perm;
stable_get_permutation(block, description, perm);

size_t columns = block.columns();
for (size_t i = 0; i < columns; ++i) {
block.safe_get_by_position(i).column =
block.safe_get_by_position(i).column->permute(perm, 0);
}
}

} // namespace doris::vectorized
Loading