diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5ce525022f3133..61a460d4084955 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1324,6 +1324,8 @@ DEFINE_mInt64(compaction_batch_size, "-1"); // filter wrong data. DEFINE_mBool(enable_parquet_page_index, "true"); +DEFINE_mBool(ignore_not_found_file_in_external_table, "true"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index a3b94705dea5a4..b30cd6265f336a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1407,6 +1407,10 @@ DECLARE_mInt64(compaction_batch_size); DECLARE_mBool(enable_parquet_page_index); +// Wheather to ignore not found file in external teble(eg, hive) +// Default is true, if set to false, the not found file will result in query failure. +DECLARE_mBool(ignore_not_found_file_in_external_table); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index f6f029b9de0c94..0c7929f0bff234 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -135,6 +135,8 @@ Status VFileScanner::prepare( _convert_to_output_block_timer = ADD_TIMER(_local_state->scanner_profile(), "FileScannerConvertOuputBlockTime"); _empty_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT); + _not_found_file_counter = + ADD_COUNTER(_local_state->scanner_profile(), "NotFoundFileNum", TUnit::UNIT); _file_counter = ADD_COUNTER(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT); _has_fully_rf_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "HasFullyRfFileNumber", TUnit::UNIT); @@ -283,9 +285,9 @@ Status VFileScanner::_get_block_wrapped(RuntimeState* state, Block* block, bool* // And the file may already be removed from storage. // Just ignore not found files. Status st = _get_next_reader(); - if (st.is()) { + if (st.is() && config::ignore_not_found_file_in_external_table) { _cur_reader_eof = true; - COUNTER_UPDATE(_empty_file_counter, 1); + COUNTER_UPDATE(_not_found_file_counter, 1); continue; } else if (!st) { return st; diff --git a/be/src/vec/exec/scan/vfile_scanner.h b/be/src/vec/exec/scan/vfile_scanner.h index 332bdfe11e11f2..fb61c5aa19e267 100644 --- a/be/src/vec/exec/scan/vfile_scanner.h +++ b/be/src/vec/exec/scan/vfile_scanner.h @@ -178,6 +178,7 @@ class VFileScanner : public VScanner { RuntimeProfile::Counter* _pre_filter_timer = nullptr; RuntimeProfile::Counter* _convert_to_output_block_timer = nullptr; RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _has_fully_rf_file_counter = nullptr;