Skip to content

[fix](iceberg) prevent coredump if read position delete file failed #34802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 1 commit into from
May 14, 2024
Merged

[fix](iceberg) prevent coredump if read position delete file failed #34802

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 9 additions & 13 deletions be/src/vec/exec/format/table/iceberg_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,17 +529,15 @@ Status IcebergParquetReader::init_reader(
_file_col_names = file_col_names;
_colname_to_value_range = colname_to_value_range;
auto parquet_meta_kv = parquet_reader->get_metadata_key_values();
static_cast<void>(_gen_col_name_maps(parquet_meta_kv));
RETURN_IF_ERROR(_gen_col_name_maps(parquet_meta_kv));
_gen_file_col_names();
_gen_new_colname_to_value_range();
parquet_reader->set_table_to_file_col_map(_table_col_to_file_col);
parquet_reader->iceberg_sanitize(_all_required_col_names);
Status status = parquet_reader->init_reader(
return parquet_reader->init_reader(
_all_required_col_names, _not_in_file_col_names, &_new_colname_to_value_range,
conjuncts, tuple_descriptor, row_descriptor, colname_to_slot_id,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);

return status;
}

Status IcebergParquetReader ::_read_position_delete_file(const TFileRangeDesc* delete_range,
Expand All @@ -556,7 +554,7 @@ Status IcebergParquetReader ::_read_position_delete_file(const TFileRangeDesc* d
std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
partition_columns;
std::unordered_map<std::string, VExprContextSPtr> missing_columns;
static_cast<void>(parquet_delete_reader.set_fill_columns(partition_columns, missing_columns));
RETURN_IF_ERROR(parquet_delete_reader.set_fill_columns(partition_columns, missing_columns));

const tparquet::FileMetaData* meta_data = parquet_delete_reader.get_meta_data();
bool dictionary_coded = true;
Expand Down Expand Up @@ -608,25 +606,23 @@ Status IcebergOrcReader::init_reader(
_gen_file_col_names();
_gen_new_colname_to_value_range();
orc_reader->set_table_col_to_file_col(_table_col_to_file_col);
Status status =
orc_reader->init_reader(&_all_required_col_names, &_new_colname_to_value_range,
conjuncts, false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
return status;
return orc_reader->init_reader(&_all_required_col_names, &_new_colname_to_value_range,
conjuncts, false, tuple_descriptor, row_descriptor,
not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts);
}

Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete_range,
DeleteFile* position_delete) {
OrcReader orc_delete_reader(_profile, _state, _params, *delete_range,
READ_DELETE_FILE_BATCH_SIZE, _state->timezone(), _io_ctx);
std::unordered_map<std::string, ColumnValueRangeType> colname_to_value_range;
Status init_status = orc_delete_reader.init_reader(
&delete_file_col_names, &colname_to_value_range, {}, false, {}, {}, nullptr, nullptr);
RETURN_IF_ERROR(orc_delete_reader.init_reader(&delete_file_col_names, &colname_to_value_range,
{}, false, {}, {}, nullptr, nullptr));

std::unordered_map<std::string, std::tuple<std::string, const SlotDescriptor*>>
partition_columns;
std::unordered_map<std::string, VExprContextSPtr> missing_columns;
static_cast<void>(orc_delete_reader.set_fill_columns(partition_columns, missing_columns));
RETURN_IF_ERROR(orc_delete_reader.set_fill_columns(partition_columns, missing_columns));

bool eof = false;
DataTypePtr data_type_file_path {new DataTypeString};
Expand Down