[Improvement](inverted index) Remove the check for inverted index file exists

be/src/olap/olap_common.h

@@ -357,7 +357,6 @@ struct OlapReaderStatistics {
     int64_t inverted_index_query_timer = 0;
     int64_t inverted_index_query_cache_hit = 0;
     int64_t inverted_index_query_cache_miss = 0;
-    int64_t inverted_index_query_file_exists_timer = 0;
     int64_t inverted_index_query_null_bitmap_timer = 0;
     int64_t inverted_index_query_bitmap_copy_timer = 0;
     int64_t inverted_index_query_bitmap_op_timer = 0;

be/src/olap/rowset/segment_v2/inverted_index_cache.cpp

@@ -136,18 +136,17 @@ Status InvertedIndexSearcherCache::get_index_searcher(const io::FileSystemSPtr&
             std::unique_ptr<MemTracker>(new MemTracker("InvertedIndexSearcherCacheWithRead"));
 #ifndef BE_TEST
     {
-        bool exists = false;
-        {
-            SCOPED_RAW_TIMER(&stats->inverted_index_query_file_exists_timer);
-            RETURN_IF_ERROR(fs->exists(file_path, &exists));
-        }
-        if (!exists) {
-            return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
-                    "inverted index path: {} not exist.", file_path);
-        }
         SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer);
         SCOPED_CONSUME_MEM_TRACKER(mem_tracker.get());
-        index_searcher = build_index_searcher(fs, index_dir, file_name);
+        try {
+            index_searcher = build_index_searcher(fs, index_dir, file_name);
+        } catch (CLuceneError& err) {
+            if (err.number() == CL_ERR_FileNotFound) {
+                return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
+                        "inverted index path: {} not exist.", file_path);
+            }
+            throw err;
+        }
     }
 #endif
 

be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp

@@ -266,30 +266,26 @@ bool DorisCompoundDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, co
     io::FileBlockCachePathPolicy cache_policy;
     auto type = config::enable_file_cache ? config::file_cache_type : "";
     io::FileReaderOptions reader_options(io::cache_type_from_string(type), cache_policy);
-    if (!fs->open_file(fd, reader_options, &h->_reader).ok()) {
-        error.set(CL_ERR_IO, "open file error");
+    auto st = fs->open_file(fd, reader_options, &h->_reader);
+    if (st.is_not_found()) {
+        error.set(CL_ERR_FileNotFound, "File does not exist");
+    } else if (st.is_io_error()) {
+        error.set(CL_ERR_IO, "File open io error");
+    } else if (st.code() == ErrorCode::PERMISSION_DENIED) {
+        error.set(CL_ERR_IO, "File Access denied");
+    } else {
+        error.set(CL_ERR_IO, "Could not open file");
     }
 
     //Check if a valid handle was retrieved
-    if (h->_reader) {
+    if (st.ok() && h->_reader) {
         //Store the file length
         h->_length = h->_reader->size();
         h->_fpos = 0;
         ret = _CLNEW FSIndexInput(h, buffer_size);
         return true;
-
-    } else {
-        int err = errno;
-        if (err == ENOENT) {
-            error.set(CL_ERR_IO, "File does not exist");
-        } else if (err == EACCES) {
-            error.set(CL_ERR_IO, "File Access denied");
-        } else if (err == EMFILE) {
-            error.set(CL_ERR_IO, "Too many open files");
-        } else {
-            error.set(CL_ERR_IO, "Could not open file");
-        }
     }
+
     delete h->_shared_lock;
     _CLDECDELETE(h)
     return false;
@@ -532,19 +528,6 @@ void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _pa
     }
 
     lucene::store::Directory::setLockFactory(lock_factory);
-
-    // It's fail checking directory existence in S3.
-    if (fs->type() == io::FileSystemType::S3) {
-        return;
-    }
-    bool exists = false;
-    LOG_AND_THROW_IF_ERROR(fs->exists(directory, &exists),
-                           "Doris compound directory init IO error");
-    if (!exists) {
-        auto e = "Doris compound directory init error: " + directory + " is not a directory";
-        LOG(WARNING) << e;
-        _CLTHROWA(CL_ERR_IO, e.c_str());
-    }
 }
 
 void DorisCompoundDirectory::priv_getFN(char* buffer, const char* name) const {
@@ -616,7 +599,11 @@ int64_t DorisCompoundDirectory::fileLength(const char* name) const {
     char buffer[CL_MAX_DIR];
     priv_getFN(buffer, name);
     int64_t size = -1;
-    LOG_AND_THROW_IF_ERROR(fs->file_size(buffer, &size), "Get file size IO error");
+    auto st = fs->file_size(buffer, &size);
+    if (st.is_not_found()) {
+        _CLTHROWA(CL_ERR_FileNotFound, "File does not exist");
+    }
+    LOG_AND_THROW_IF_ERROR(st, "Get file size IO error");
     return size;
 }
 

be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h

@@ -54,7 +54,7 @@ class CLUCENE_EXPORT DorisCompoundReader : public lucene::store::Directory {
     lucene::store::RAMDirectory* ram_dir;
     std::string directory;
     std::string file_name;
-    CL_NS(store)::IndexInput* stream;
+    CL_NS(store)::IndexInput* stream = nullptr;
 
     using EntriesType =
             lucene::util::CLHashMap<char*, ReaderFileEntry*, lucene::util::Compare::Char,

be/src/olap/rowset/segment_v2/inverted_index_reader.cpp

@@ -99,13 +99,6 @@ bool InvertedIndexReader::_is_match_query(InvertedIndexQueryType query_type) {
             query_type == InvertedIndexQueryType::MATCH_REGEXP_QUERY);
 }
 
-bool InvertedIndexReader::indexExists(io::Path& index_file_path) {
-    // SCOPED_RAW_TIMER(&stats->inverted_index_query_file_exists_timer);
-    bool exists = false;
-    RETURN_IF_ERROR(_fs->exists(index_file_path, &exists));
-    return exists;
-}
-
 std::unique_ptr<lucene::analysis::Analyzer> InvertedIndexReader::create_analyzer(
         InvertedIndexCtx* inverted_index_ctx) {
     std::unique_ptr<lucene::analysis::Analyzer> analyzer;
@@ -210,21 +203,18 @@ Status InvertedIndexReader::read_null_bitmap(OlapReaderStatistics* stats,
             return Status::OK();
         }
 
-        bool exists = false;
-        {
-            SCOPED_RAW_TIMER(&stats->inverted_index_query_file_exists_timer);
-            RETURN_IF_ERROR(_fs->exists(index_file_path, &exists));
-        }
-        if (!exists) {
-            LOG(WARNING) << "inverted index: " << index_file_path.native() << " not exist.";
-            return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
-                    "inverted index path: {} not exist.", index_file_path.native());
-        }
-
         if (!dir) {
-            dir = new DorisCompoundReader(
-                    DorisCompoundDirectoryFactory::getDirectory(_fs, index_dir.c_str()),
-                    index_file_name.c_str(), config::inverted_index_read_buffer_size);
+            try {
+                dir = new DorisCompoundReader(
+                        DorisCompoundDirectoryFactory::getDirectory(_fs, index_dir.c_str()),
+                        index_file_name.c_str(), config::inverted_index_read_buffer_size);
+            } catch (CLuceneError& err) {
+                if (err.number() == CL_ERR_FileNotFound) {
+                    return Status::Error<ErrorCode::INVERTED_INDEX_FILE_NOT_FOUND>(
+                            "inverted index path: {} not exist.", index_file_path.native());
+                }
+                throw err;
+            }
             owned_dir = true;
         }
 
@@ -763,17 +753,18 @@ BkdIndexReader::BkdIndexReader(io::FileSystemSPtr fs, const std::string& path,
     auto index_dir = io_path.parent_path();
     auto index_file_name = InvertedIndexDescriptor::get_index_file_name(io_path.filename(),
                                                                         index_meta->index_id());
-
-    // check index file existence
     auto index_file = index_dir / index_file_name;
-    if (!indexExists(index_file)) {
-        LOG(WARNING) << "bkd index: " << index_file.string() << " not exist.";
-        return;
-    }
     _file_full_path = index_file;
-    _compoundReader = std::make_unique<DorisCompoundReader>(
-            DorisCompoundDirectoryFactory::getDirectory(fs, index_dir.c_str()),
-            index_file_name.c_str(), config::inverted_index_read_buffer_size);
+    try {
+        _compoundReader = std::make_unique<DorisCompoundReader>(
+                DorisCompoundDirectoryFactory::getDirectory(fs, index_dir.c_str()),
+                index_file_name.c_str(), config::inverted_index_read_buffer_size);
+    } catch (CLuceneError& err) {
+        if (err.number() == CL_ERR_FileNotFound) {
+            LOG(WARNING) << "bkd index: " << index_file.string() << " not exist.";
+            return;
+        }
+    }
 }
 
 Status BkdIndexReader::new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,

be/src/vec/exec/scan/new_olap_scan_node.cpp

@@ -173,8 +173,6 @@ Status NewOlapScanNode::_init_profile() {
     _inverted_index_query_timer = ADD_TIMER(_segment_profile, "InvertedIndexQueryTime");
     _inverted_index_query_null_bitmap_timer =
             ADD_TIMER(_segment_profile, "InvertedIndexQueryNullBitmapTime");
-    _inverted_index_query_file_exists_timer =
-            ADD_TIMER(_segment_profile, "InvertedIndexQueryFileExistsTime");
     _inverted_index_query_bitmap_copy_timer =
             ADD_TIMER(_segment_profile, "InvertedIndexQueryBitmapCopyTime");
     _inverted_index_query_bitmap_op_timer =

be/src/vec/exec/scan/new_olap_scan_node.h

@@ -191,7 +191,6 @@ class NewOlapScanNode : public VScanNode {
     RuntimeProfile::Counter* _inverted_index_query_cache_hit_counter = nullptr;
     RuntimeProfile::Counter* _inverted_index_query_cache_miss_counter = nullptr;
     RuntimeProfile::Counter* _inverted_index_query_timer = nullptr;
-    RuntimeProfile::Counter* _inverted_index_query_file_exists_timer = nullptr;
     RuntimeProfile::Counter* _inverted_index_query_null_bitmap_timer = nullptr;
     RuntimeProfile::Counter* _inverted_index_query_bitmap_copy_timer = nullptr;
     RuntimeProfile::Counter* _inverted_index_query_bitmap_op_timer = nullptr;

be/src/vec/exec/scan/new_olap_scanner.cpp

@@ -587,8 +587,6 @@ void NewOlapScanner::_update_counters_before_close() {
     COUNTER_UPDATE(olap_parent->_inverted_index_query_cache_miss_counter,
                    stats.inverted_index_query_cache_miss);
     COUNTER_UPDATE(olap_parent->_inverted_index_query_timer, stats.inverted_index_query_timer);
-    COUNTER_UPDATE(olap_parent->_inverted_index_query_file_exists_timer,
-                   stats.inverted_index_query_file_exists_timer);
     COUNTER_UPDATE(olap_parent->_inverted_index_query_null_bitmap_timer,
                    stats.inverted_index_query_null_bitmap_timer);
     COUNTER_UPDATE(olap_parent->_inverted_index_query_bitmap_copy_timer,