Skip to content

[fix](iceberg_orc)Fixed the bug that the iceberg reader did not perform position delete when reading the orc file without a predicate. #34814

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 1 commit into from
May 15, 2024
Merged

[fix](iceberg_orc)Fixed the bug that the iceberg reader did not perform position delete when reading the orc file without a predicate. #34814

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions be/src/vec/exec/format/orc/vorc_reader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1747,9 +1747,16 @@ Status OrcReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
}
} else {
if (_delete_rows_filter_ptr) {
_execute_filter_position_delete_rowids(*_delete_rows_filter_ptr);
RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(block, columns_to_filter,
(*_delete_rows_filter_ptr)));
} else {
std::unique_ptr<IColumn::Filter> filter(new IColumn::Filter(block->rows(), 1));
Copy link
Member

@AshinGau AshinGau May 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why need to create filter when there's no delete rows?

Copy link
Contributor Author

@hubgeter hubgeter May 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_delete_rows_filter_ptr be used in transactional hive orc , not position delete.

_execute_filter_position_delete_rowids(*filter);
RETURN_IF_CATCH_EXCEPTION(
Block::filter_block_internal(block, columns_to_filter, (*filter)));
}

Block::erase_useless_column(block, column_to_keep);
static_cast<void>(_convert_dict_cols_to_string_cols(block, &batch_vec));
}
Expand Down
56 changes: 56 additions & 0 deletions regression-test/suites/external_table_p2/iceberg/iceberg_position_delete.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,62 @@ suite("iceberg_position_delete", "p2,external,iceberg,external_remote,external_r
qt_parquet_19 """ select count(*) from iceberg_position_parquet where name != 'final entryxxxxxx' ;"""
qt_parquet_20 """ select count(*) from iceberg_position_parquet; """


List<List<Object>> iceberg_position_orc = sql """ select * from iceberg_position_orc ;"""
List<List<Object>> iceberg_position_parquet = sql """ select * from iceberg_position_parquet;"""
List<List<Object>> iceberg_position_gen = sql """ select * from iceberg_position_gen_data;"""

assertTrue(iceberg_position_orc.size() == iceberg_position_gen.size())
assertTrue(iceberg_position_orc.size() == iceberg_position_parquet.size())
assertTrue(iceberg_position_orc.size() == 5632)


List<List<Object>> iceberg_position_orc_1 = sql """select * from iceberg_position_orc where id != 1;"""
List<List<Object>> iceberg_position_orc_2 = sql """select * from iceberg_position_orc where name != "hello word" ;"""
List<List<Object>> iceberg_position_orc_3 = sql """select id from iceberg_position_orc where id != 1;"""
List<List<Object>> iceberg_position_orc_4 = sql """select name from iceberg_position_orc where id != 1;"""
List<List<Object>> iceberg_position_orc_5 = sql """select name from iceberg_position_orc where name != "hello word" ;"""
List<List<Object>> iceberg_position_orc_6 = sql """select id from iceberg_position_orc where name != "hello word" ;"""
List<List<Object>> iceberg_position_orc_7 = sql """select * from iceberg_position_orc where id != 1 and name != "33333";"""
assertTrue(iceberg_position_orc_1.size() == 5632)
assertTrue(iceberg_position_orc_2.size() == 5632)
assertTrue(iceberg_position_orc_3.size() == 5632)
assertTrue(iceberg_position_orc_4.size() == 5632)
assertTrue(iceberg_position_orc_5.size() == 5632)
assertTrue(iceberg_position_orc_6.size() == 5632)
assertTrue(iceberg_position_orc_7.size() == 5632)


List<List<Object>> iceberg_position_gen_1 = sql """select * from iceberg_position_gen_data where id != 1 and name != "hello";"""
assertTrue(iceberg_position_gen_1.size() == 5632)

List<List<Object>> iceberg_position_gen_2 = sql """select * from iceberg_position_gen_data where id != 2;"""
assertTrue(iceberg_position_gen_2.size() == 5120)

List<List<Object>> iceberg_position_gen_22 = sql """select * from iceberg_position_gen_data where id != 5;"""
assertTrue(iceberg_position_gen_22.size() == 5632)

List<List<Object>> iceberg_position_gen_3 = sql """select * from iceberg_position_gen_data where name != "hello word" ;"""
assertTrue(iceberg_position_gen_3.size() == 5632)

List<List<Object>> iceberg_position_gen_4 = sql """select id from iceberg_position_gen_data where id != 2;"""
assertTrue(iceberg_position_gen_4.size() == 5120)

List<List<Object>> iceberg_position_gen_44 = sql """select id from iceberg_position_gen_data where id != 5;"""
assertTrue(iceberg_position_gen_44.size() == 5632)

List<List<Object>> iceberg_position_gen_5 = sql """select name from iceberg_position_gen_data where id != 2;"""
assertTrue(iceberg_position_gen_5.size() == 5120)

List<List<Object>> iceberg_position_gen_55 = sql """select name from iceberg_position_gen_data where id != 5;"""
assertTrue(iceberg_position_gen_55.size() == 5632)

List<List<Object>> iceberg_position_gen_6 = sql """select name from iceberg_position_gen_data where name != "hello wordxx" ;"""
assertTrue(iceberg_position_gen_6.size() == 5632)

List<List<Object>> iceberg_position_gen_7 = sql """select id from iceberg_position_gen_data where name != "hello word" ;"""
assertTrue(iceberg_position_gen_7.size() == 5632)

sql """drop catalog ${catalog_name}"""
}
}
Expand Down