From 4886c4342cfebbd915bdea6595e77e3f2e13df39 Mon Sep 17 00:00:00 2001 From: csun5285 Date: Thu, 27 Jun 2024 16:14:53 +0800 Subject: [PATCH] fix exists add clucene fix v2 remove unused code add test --- be/src/clucene | 2 +- .../segment_v2/inverted_index_file_reader.cpp | 20 +++++-- .../inverted_index_fs_directory.cpp | 49 +++++++-------- .../segment_v2/inverted_index_reader.cpp | 15 +---- .../rowset/segment_v2/inverted_index_reader.h | 2 - .../test_index_not_found_fault_injection.out | 13 ++++ ...est_index_not_found_fault_injection.groovy | 59 +++++++++++++++++++ 7 files changed, 108 insertions(+), 52 deletions(-) create mode 100644 regression-test/data/fault_injection_p0/test_index_not_found_fault_injection.out create mode 100644 regression-test/suites/fault_injection_p0/test_index_not_found_fault_injection.groovy diff --git a/be/src/clucene b/be/src/clucene index dd200e10e72120..5db9db68e448b8 160000 --- a/be/src/clucene +++ b/be/src/clucene @@ -1 +1 @@ -Subproject commit dd200e10e72120445bd897f3dcc515702f4dc80b +Subproject commit 5db9db68e448b8ccfd360d02666bbac44e6f8d1a diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp index 7a744ea939e02e..dbd86bb93a511e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp @@ -23,6 +23,7 @@ #include "olap/rowset/segment_v2/inverted_index_compound_reader.h" #include "olap/rowset/segment_v2/inverted_index_fs_directory.h" #include "olap/tablet_schema.h" +#include "util/debug_points.h" namespace doris::segment_v2 { @@ -41,14 +42,17 @@ Status InvertedIndexFileReader::_init_from_v2(int32_t read_buffer_size) { std::unique_lock lock(_mutex); // Lock for writing try { - bool exists = false; - RETURN_IF_ERROR(_fs->exists(index_file_full_path, &exists)); - if (!exists) { + int64_t file_size = 0; + Status st = _fs->file_size(index_file_full_path, &file_size); + DBUG_EXECUTE_IF("inverted file read error: index file not found", { + st = Status::Error("index file not found"); + }) + if (st.code() == ErrorCode::NOT_FOUND) { return Status::Error( "inverted index file {} is not found", index_file_full_path); + } else if (!st.ok()) { + return st; } - int64_t file_size = 0; - RETURN_IF_ERROR(_fs->file_size(index_file_full_path, &file_size)); if (file_size == 0) { LOG(WARNING) << "inverted index file " << index_file_full_path << " is empty."; return Status::Error( @@ -157,6 +161,10 @@ Result> InvertedIndexFileReader::_open( dir->close(); _CLDELETE(dir) } + if (err.number() == CL_ERR_FileNotFound) { + return ResultError(Status::Error( + "inverted index path: {} not exist.", index_file_path)); + } return ResultError(Status::Error( "CLuceneError occur when open idx file {}, error msg: {}", index_file_path, err.what())); @@ -174,7 +182,7 @@ Result> InvertedIndexFileReader::_open( if (index_it == _indices_entries.end()) { std::ostringstream errMsg; errMsg << "No index with id " << index_id << " found"; - return ResultError(Status::Error( + return ResultError(Status::Error( "CLuceneError occur when open idx file {}, error msg: {}", InvertedIndexDescriptor::get_index_file_path_v2(_index_path_prefix), errMsg.str())); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp index 499a3e41d9a887..54d484d119941b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp @@ -112,30 +112,28 @@ bool DorisFSDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, const ch reader_options.cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE : io::FileCachePolicy::NO_CACHE; reader_options.is_doris_table = true; - if (!fs->open_file(path, &h->_reader, &reader_options).ok()) { - error.set(CL_ERR_IO, "open file error"); + Status st = fs->open_file(path, &h->_reader, &reader_options); + DBUG_EXECUTE_IF("inverted file read error: index file not found", + { st = Status::Error("index file not found"); }) + if (st.code() == ErrorCode::NOT_FOUND) { + error.set(CL_ERR_FileNotFound, "File does not exist"); + } else if (st.code() == ErrorCode::IO_ERROR) { + error.set(CL_ERR_IO, "File open io error"); + } else if (st.code() == ErrorCode::PERMISSION_DENIED) { + error.set(CL_ERR_IO, "File Access denied"); + } else { + error.set(CL_ERR_IO, "Could not open file"); } //Check if a valid handle was retrieved - if (h->_reader) { + if (st.ok() && h->_reader) { //Store the file length h->_length = h->_reader->size(); h->_fpos = 0; ret = _CLNEW FSIndexInput(std::move(h), buffer_size); return true; - - } else { - int err = errno; - if (err == ENOENT) { - error.set(CL_ERR_IO, "File does not exist"); - } else if (err == EACCES) { - error.set(CL_ERR_IO, "File Access denied"); - } else if (err == EMFILE) { - error.set(CL_ERR_IO, "Too many open files"); - } else { - error.set(CL_ERR_IO, "Could not open file"); - } } + //delete h->_shared_lock; //_CLDECDELETE(h) return false; @@ -349,19 +347,6 @@ void DorisFSDirectory::init(const io::FileSystemSPtr& fs, const char* path, } lucene::store::Directory::setLockFactory(lock_factory); - - // It's fail checking directory existence in S3. - if (_fs->type() == io::FileSystemType::S3) { - return; - } - bool exists = false; - LOG_AND_THROW_IF_ERROR(_fs->exists(directory, &exists), - "Doris compound directory init IO error"); - if (!exists) { - auto e = "Doris compound directory init error: " + directory + " is not a directory"; - LOG(WARNING) << e; - _CLTHROWA(CL_ERR_IO, e.c_str()); - } } void DorisFSDirectory::priv_getFN(char* buffer, const char* name) const { @@ -432,7 +417,13 @@ int64_t DorisFSDirectory::fileLength(const char* name) const { char buffer[CL_MAX_DIR]; priv_getFN(buffer, name); int64_t size = -1; - LOG_AND_THROW_IF_ERROR(_fs->file_size(buffer, &size), "Get file size IO error"); + Status st = _fs->file_size(buffer, &size); + DBUG_EXECUTE_IF("inverted file read error: index file not found", + { st = Status::Error("index file not found"); }) + if (st.code() == ErrorCode::NOT_FOUND) { + _CLTHROWA(CL_ERR_FileNotFound, "File does not exist"); + } + LOG_AND_THROW_IF_ERROR(st, "Get file size IO error"); return size; } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index a91dfe169d5d46..57479add2f7e6a 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -201,8 +201,6 @@ Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach return Status::OK(); } - RETURN_IF_ERROR(check_file_exist(index_file_key)); - if (!dir) { // TODO: ugly code here, try to refact. auto directory = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta)); @@ -252,7 +250,7 @@ Status InvertedIndexReader::handle_searcher_cache( auto mem_tracker = std::make_unique("InvertedIndexSearcherCacheWithRead"); SCOPED_RAW_TIMER(&stats->inverted_index_searcher_open_timer); IndexSearcherPtr searcher; - RETURN_IF_ERROR(check_file_exist(index_file_key)); + auto dir = DORIS_TRY(_inverted_index_file_reader->open(&_index_meta)); // try to reuse index_searcher's directory to read null_bitmap to cache // to avoid open directory additionally for null_bitmap @@ -286,17 +284,6 @@ Status InvertedIndexReader::create_index_searcher(lucene::store::Directory* dir, return Status::OK(); }; -Status InvertedIndexReader::check_file_exist(const std::string& index_file_key) { - bool exists = false; - RETURN_IF_ERROR(_inverted_index_file_reader->index_file_exist(&_index_meta, &exists)); - if (!exists) { - LOG(WARNING) << "inverted index: " << index_file_key << " not exist."; - return Status::Error( - "inverted index input file {} not found", index_file_key); - } - return Status::OK(); -} - Status FullTextIndexReader::new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state, std::unique_ptr* iterator) { *iterator = InvertedIndexIterator::create_unique(stats, runtime_state, shared_from_this()); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index 9d603a85d12414..90df92f0728a9b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -141,8 +141,6 @@ class InvertedIndexReader : public std::enable_shared_from_this _inverted_index_file_reader; diff --git a/regression-test/data/fault_injection_p0/test_index_not_found_fault_injection.out b/regression-test/data/fault_injection_p0/test_index_not_found_fault_injection.out new file mode 100644 index 00000000000000..e755b8cf9d83e0 --- /dev/null +++ b/regression-test/data/fault_injection_p0/test_index_not_found_fault_injection.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +3 + +-- !sql -- +3 + +-- !sql -- +3 + +-- !sql -- +3 + diff --git a/regression-test/suites/fault_injection_p0/test_index_not_found_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_not_found_fault_injection.groovy new file mode 100644 index 00000000000000..6b27fa9b2b3924 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_index_not_found_fault_injection.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_index_not_found_fault_injection", "nonConcurrent") { + def testTable = "test_index_not_found" + + sql "DROP TABLE IF EXISTS ${testTable}" + sql """ + CREATE TABLE ${testTable} ( + `@timestamp` int(11) NULL COMMENT "", + `clientip` string NULL COMMENT "", + `request` string NULL COMMENT "", + `status` string NULL COMMENT "", + `size` string NULL COMMENT "", + INDEX clientip_idx (`clientip`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '', + INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '', + INDEX status_idx (`status`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '', + INDEX size_idx (`size`) USING INVERTED PROPERTIES("parser" = "unicode", "support_phrase" = "true") COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + + sql """ INSERT INTO ${testTable} VALUES (893964617, '40.135.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 24736); """ + sql """ INSERT INTO ${testTable} VALUES (893964653, '232.0.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 200, 3781); """ + sql """ INSERT INTO ${testTable} VALUES (893964672, '26.1.0.0', 'GET /images/hm_bg.jpg HTTP/1.0', 304, 0); """ + + try { + GetDebugPoint().enableDebugPointForAllBEs("inverted file read error: index file not found") + + qt_sql """ select count() from ${testTable} where (request match_phrase 'http'); """ + qt_sql """ select count() from ${testTable} where (request match_phrase_prefix 'http'); """ + + qt_sql """ select count() from ${testTable} where (clientip match_phrase 'http' or request match_phrase 'http' or status match_phrase 'http' or size match_phrase 'http'); """ + qt_sql """ select count() from ${testTable} where (clientip match_phrase_prefix 'http' or request match_phrase_prefix 'http' or status match_phrase_prefix 'http' or size match_phrase_prefix 'http'); """ + } finally { + GetDebugPoint().disableDebugPointForAllBEs("inverted file read error: index file not found") + } +} \ No newline at end of file