From cd8fc3031a76610a3fe008b054ab83b469dc5d3f Mon Sep 17 00:00:00 2001 From: freemandealer Date: Wed, 16 Oct 2024 21:15:26 +0800 Subject: [PATCH 1/2] [enhancement](cloud) fix TTL cache size negative overflow Signed-off-by: freemandealer --- be/src/io/cache/block_file_cache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 321d9d73d89ed9..adfa94bc84f1f1 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -1129,7 +1129,8 @@ void BlockFileCache::reset_range(const UInt128Wrapper& hash, size_t offset, size _files.find(hash)->second.find(offset) != _files.find(hash)->second.end()); FileBlockCell* cell = get_cell(hash, offset, cache_lock); DCHECK(cell != nullptr); - if (cell->file_block->cache_type() != FileCacheType::TTL) { + if (cell->file_block->cache_type() != FileCacheType::TTL || + config::enable_ttl_cache_evict_using_lru) { auto& queue = get_queue(cell->file_block->cache_type()); DCHECK(queue.contains(hash, offset, cache_lock)); auto iter = queue.get(hash, offset, cache_lock); From a36a42a50f05bb1aed1801eac70ea595434a2d03 Mon Sep 17 00:00:00 2001 From: freemandealer Date: Thu, 17 Oct 2024 12:23:54 +0800 Subject: [PATCH 2/2] response to the reviewer Signed-off-by: freemandealer --- be/src/io/cache/block_file_cache.cpp | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index adfa94bc84f1f1..998b31ff49366c 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -243,15 +243,12 @@ void BlockFileCache::use_cell(const FileBlockCell& cell, FileBlocks* result, boo result->push_back(cell.file_block); } - if (cell.file_block->cache_type() != FileCacheType::TTL || - config::enable_ttl_cache_evict_using_lru) { - auto& queue = get_queue(cell.file_block->cache_type()); - DCHECK(cell.queue_iterator) << "impossible"; - /// Move to the end of the queue. The iterator remains valid. - if (move_iter_flag) { - queue.move_to_end(*cell.queue_iterator, cache_lock); - } + auto& queue = get_queue(cell.file_block->cache_type()); + /// Move to the end of the queue. The iterator remains valid. + if (cell.queue_iterator && move_iter_flag) { + queue.move_to_end(*cell.queue_iterator, cache_lock); } + cell.update_atime(); cell.is_deleted = false; } @@ -357,7 +354,7 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper& hash, const CacheConte auto st = cell.file_block->change_cache_type_between_ttl_and_others( FileCacheType::NORMAL); if (st.ok()) { - if (config::enable_ttl_cache_evict_using_lru) { + if (cell.queue_iterator) { auto& ttl_queue = get_queue(FileCacheType::TTL); ttl_queue.remove(cell.queue_iterator.value(), cache_lock); } @@ -1052,7 +1049,7 @@ bool BlockFileCache::remove_if_ttl_file_unlock(const UInt128Wrapper& file_key, b auto st = cell.file_block->change_cache_type_between_ttl_and_others( FileCacheType::NORMAL); if (st.ok()) { - if (config::enable_ttl_cache_evict_using_lru) { + if (cell.queue_iterator) { ttl_queue.remove(cell.queue_iterator.value(), cache_lock); } auto& queue = get_queue(FileCacheType::NORMAL); @@ -1129,8 +1126,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper& hash, size_t offset, size _files.find(hash)->second.find(offset) != _files.find(hash)->second.end()); FileBlockCell* cell = get_cell(hash, offset, cache_lock); DCHECK(cell != nullptr); - if (cell->file_block->cache_type() != FileCacheType::TTL || - config::enable_ttl_cache_evict_using_lru) { + if (cell->queue_iterator) { auto& queue = get_queue(cell->file_block->cache_type()); DCHECK(queue.contains(hash, offset, cache_lock)); auto iter = queue.get(hash, offset, cache_lock); @@ -1806,10 +1802,8 @@ std::string BlockFileCache::clear_file_cache_directly() { << " time_elapsed=" << duration_cast(steady_clock::now() - start).count() << " num_files=" << num_files << " cache_size=" << cache_size << " index_queue_size=" << index_queue_size << " normal_queue_size=" << normal_queue_size - << " disposible_queue_size=" << disposible_queue_size; - if (config::enable_ttl_cache_evict_using_lru) { - ss << "ttl_queue_size=" << ttl_queue_size; - } + << " disposible_queue_size=" << disposible_queue_size << "ttl_queue_size=" << ttl_queue_size; + auto msg = ss.str(); LOG(INFO) << msg; return msg;