From 04d1dc4037740b296e80f5a1688e05066d39a2bd Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 26 Jun 2024 14:38:53 +0800 Subject: [PATCH 1/3] [opt](file-scanner) add not found file number in profile --- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 4 ++++ be/src/vec/exec/scan/vfile_scanner.cpp | 5 +++-- be/src/vec/exec/scan/vfile_scanner.h | 1 + 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5ce525022f3133..61a460d4084955 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1324,6 +1324,8 @@ DEFINE_mInt64(compaction_batch_size, "-1"); // filter wrong data. DEFINE_mBool(enable_parquet_page_index, "true"); +DEFINE_mBool(ignore_not_found_file_in_external_table, "true"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index a3b94705dea5a4..b30cd6265f336a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1407,6 +1407,10 @@ DECLARE_mInt64(compaction_batch_size); DECLARE_mBool(enable_parquet_page_index); +// Wheather to ignore not found file in external teble(eg, hive) +// Default is true, if set to false, the not found file will result in query failure. +DECLARE_mBool(ignore_not_found_file_in_external_table); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index f6f029b9de0c94..2dfad048a7a918 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -135,6 +135,7 @@ Status VFileScanner::prepare( _convert_to_output_block_timer = ADD_TIMER(_local_state->scanner_profile(), "FileScannerConvertOuputBlockTime"); _empty_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT); + _not_found_file_counter = ADD_COUNTER(_parent->_scanner_profile, "NotFoundFileNum", TUnit::UNIT); _file_counter = ADD_COUNTER(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT); _has_fully_rf_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "HasFullyRfFileNumber", TUnit::UNIT); @@ -283,9 +284,9 @@ Status VFileScanner::_get_block_wrapped(RuntimeState* state, Block* block, bool* // And the file may already be removed from storage. // Just ignore not found files. Status st = _get_next_reader(); - if (st.is()) { + if (st.is() && config::ignore_not_found_file_in_external_table) { _cur_reader_eof = true; - COUNTER_UPDATE(_empty_file_counter, 1); + COUNTER_UPDATE(_not_found_file_counter, 1); continue; } else if (!st) { return st; diff --git a/be/src/vec/exec/scan/vfile_scanner.h b/be/src/vec/exec/scan/vfile_scanner.h index 332bdfe11e11f2..fb61c5aa19e267 100644 --- a/be/src/vec/exec/scan/vfile_scanner.h +++ b/be/src/vec/exec/scan/vfile_scanner.h @@ -178,6 +178,7 @@ class VFileScanner : public VScanner { RuntimeProfile::Counter* _pre_filter_timer = nullptr; RuntimeProfile::Counter* _convert_to_output_block_timer = nullptr; RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _has_fully_rf_file_counter = nullptr; From 03393e5515767379d344343e2274eabe134153b4 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 30 Jun 2024 22:44:01 +0800 Subject: [PATCH 2/3] format --- be/src/vec/exec/scan/vfile_scanner.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 2dfad048a7a918..a68f5b8aea3ae1 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -135,7 +135,8 @@ Status VFileScanner::prepare( _convert_to_output_block_timer = ADD_TIMER(_local_state->scanner_profile(), "FileScannerConvertOuputBlockTime"); _empty_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT); - _not_found_file_counter = ADD_COUNTER(_parent->_scanner_profile, "NotFoundFileNum", TUnit::UNIT); + _not_found_file_counter = + ADD_COUNTER(_parent->_scanner_profile, "NotFoundFileNum", TUnit::UNIT); _file_counter = ADD_COUNTER(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT); _has_fully_rf_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "HasFullyRfFileNumber", TUnit::UNIT); From 6f1ceea8f675eca3275b865f52b04796a81ae0c2 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 30 Jun 2024 23:31:56 +0800 Subject: [PATCH 3/3] format --- be/src/vec/exec/scan/vfile_scanner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index a68f5b8aea3ae1..0c7929f0bff234 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -136,7 +136,7 @@ Status VFileScanner::prepare( ADD_TIMER(_local_state->scanner_profile(), "FileScannerConvertOuputBlockTime"); _empty_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT); _not_found_file_counter = - ADD_COUNTER(_parent->_scanner_profile, "NotFoundFileNum", TUnit::UNIT); + ADD_COUNTER(_local_state->scanner_profile(), "NotFoundFileNum", TUnit::UNIT); _file_counter = ADD_COUNTER(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT); _has_fully_rf_file_counter = ADD_COUNTER(_local_state->scanner_profile(), "HasFullyRfFileNumber", TUnit::UNIT);