From e599713a2f392d369dffe90a51efeea3a536edb3 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Jul 2024 17:34:35 +0800 Subject: [PATCH 1/2] [fix](external) record not found file number --- be/src/vec/exec/format/orc/vorc_reader.cpp | 4 +++- be/src/vec/exec/scan/vfile_scanner.cpp | 16 +++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 9c016a4565e4e8..aa6f9f5f5c784c 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -261,7 +261,9 @@ Status OrcReader::_create_file_reader() { if (_io_ctx && _io_ctx->should_stop && _err_msg == "stop") { return Status::EndOfFile("stop"); } - if (_err_msg.find("No such file or directory") != std::string::npos) { + // one for fs, the other is for oss. + if (_err_msg.find("No such file or directory") != std::string::npos + || _err_msg.find("NoSuchKey") != std::string::npos) { return Status::NotFound(_err_msg); } return Status::InternalError("Init OrcReader failed. reason = {}", _err_msg); diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 3ef07c50b64b3d..0446f875817335 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -939,13 +939,19 @@ Status VFileScanner::_get_next_reader() { } COUNTER_UPDATE(_file_counter, 1); - if (init_status.is() || init_status.is()) { - // The VFileScanner for external table may try to open not exist files, - // Because FE file cache for external table may out of date. - // So, NOT_FOUND for VFileScanner is not a fail case. - // Will remove this after file reader refactor. + // The VFileScanner for external table may try to open not exist files, + // Because FE file cache for external table may out of date. + // So, NOT_FOUND for VFileScanner is not a fail case. + // Will remove this after file reader refactor. + if (init_status.is()) { COUNTER_UPDATE(_empty_file_counter, 1); continue; + } else if (init_status.is()) { + if (config::ignore_not_found_file_in_external_table) { + COUNTER_UPDATE(_not_found_file_counter, 1); + continue; + } + return Status::InternalError("failed to find reader, err: {}", init_status.to_string()); } else if (!init_status.ok()) { return Status::InternalError("failed to init reader, err: {}", init_status.to_string()); } From 21640132ebacd8db92605c4d80886cf63db751d4 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Jul 2024 17:38:36 +0800 Subject: [PATCH 2/2] 1 --- be/src/vec/exec/format/orc/vorc_reader.cpp | 4 ++-- be/src/vec/exec/scan/vfile_scanner.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index aa6f9f5f5c784c..e2ba3a57be8c9a 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -262,8 +262,8 @@ Status OrcReader::_create_file_reader() { return Status::EndOfFile("stop"); } // one for fs, the other is for oss. - if (_err_msg.find("No such file or directory") != std::string::npos - || _err_msg.find("NoSuchKey") != std::string::npos) { + if (_err_msg.find("No such file or directory") != std::string::npos || + _err_msg.find("NoSuchKey") != std::string::npos) { return Status::NotFound(_err_msg); } return Status::InternalError("Init OrcReader failed. reason = {}", _err_msg); diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 0446f875817335..e066905895e6a2 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -951,7 +951,7 @@ Status VFileScanner::_get_next_reader() { COUNTER_UPDATE(_not_found_file_counter, 1); continue; } - return Status::InternalError("failed to find reader, err: {}", init_status.to_string()); + return Status::InternalError("failed to find reader, err: {}", init_status.to_string()); } else if (!init_status.ok()) { return Status::InternalError("failed to init reader, err: {}", init_status.to_string()); }