diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp index a7a34fb87906af..5d9da0bbeb3bfa 100644 --- a/be/src/olap/rowset/rowset.cpp +++ b/be/src/olap/rowset/rowset.cpp @@ -22,6 +22,7 @@ #include "common/config.h" #include "io/cache/block_file_cache_factory.h" #include "olap/olap_define.h" +#include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/segment_loader.h" #include "olap/tablet_schema.h" #include "util/time.h" @@ -126,6 +127,14 @@ void Rowset::clear_cache() { auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key); file_cache->remove_if_cached_async(file_key); } + + // inverted index + auto file_names = get_index_file_names(); + for (const auto& file_name : file_names) { + auto file_key = io::BlockFileCache::hash(file_name); + auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key); + file_cache->remove_if_cached_async(file_key); + } } } @@ -163,4 +172,24 @@ void Rowset::merge_rowset_meta(const RowsetMeta& other) { _schema = _rowset_meta->tablet_schema(); } +std::vector Rowset::get_index_file_names() { + std::vector file_names; + auto idx_version = _schema->get_inverted_index_storage_format(); + for (int64_t seg_id = 0; seg_id < num_segments(); ++seg_id) { + if (idx_version == InvertedIndexStorageFormatPB::V1) { + for (const auto& index : _schema->inverted_indexes()) { + auto file_name = segment_v2::InvertedIndexDescriptor::get_index_file_name_v1( + rowset_id().to_string(), seg_id, index->index_id(), + index->get_index_suffix()); + file_names.emplace_back(std::move(file_name)); + } + } else { + auto file_name = segment_v2::InvertedIndexDescriptor::get_index_file_name_v2( + rowset_id().to_string(), seg_id); + file_names.emplace_back(std::move(file_name)); + } + } + return file_names; +} + } // namespace doris diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 01f321728f1bea..f70a1a6df86aa8 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -312,6 +312,8 @@ class Rowset : public std::enable_shared_from_this, public MetadataAdder Result segment_path(int64_t seg_id); + std::vector get_index_file_names(); + protected: friend class RowsetFactory; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp index 101aec40b6eb2d..4812a2b3037b85 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp @@ -77,4 +77,17 @@ std::string InvertedIndexDescriptor::get_index_file_cache_key(std::string_view i return fmt::format("{}_{}{}", index_path_prefix, index_id, suffix); } +std::string InvertedIndexDescriptor::get_index_file_name_v1(const std::string& rowset_id, + int64_t seg_id, int64_t index_id, + std::string_view index_path_suffix) { + std::string suffix = + index_path_suffix.empty() ? "" : std::string {"@"} + index_path_suffix.data(); + return fmt::format("{}_{}_{}{}{}", rowset_id, seg_id, index_id, suffix, index_suffix); +} + +std::string InvertedIndexDescriptor::get_index_file_name_v2(const std::string& rowset_id, + int64_t seg_id) { + return fmt::format("{}_{}{}", rowset_id, seg_id, index_suffix); +} + } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h b/be/src/olap/rowset/segment_v2/inverted_index_desc.h index f421c7f37903ae..23c85bcf69741f 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h @@ -50,6 +50,11 @@ class InvertedIndexDescriptor { int64_t index_id, std::string_view index_path_suffix); + static std::string get_index_file_name_v1(const std::string& rowset_id, int64_t seg_id, + int64_t index_id, std::string_view index_path_suffix); + + static std::string get_index_file_name_v2(const std::string& rowset_id, int64_t seg_id); + static const char* get_temporary_null_bitmap_file_name() { return "null_bitmap"; } static const char* get_temporary_bkd_index_data_file_name() { return "bkd"; } static const char* get_temporary_bkd_index_meta_file_name() { return "bkd_meta"; } diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp index 2e13436b3d3337..a1e6a0ec633667 100644 --- a/be/test/olap/rowset/beta_rowset_test.cpp +++ b/be/test/olap/rowset/beta_rowset_test.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,7 @@ #include "io/fs/local_file_system.h" #include "io/fs/s3_file_system.h" #include "io/fs/s3_obj_storage_client.h" +#include "json2pb/json_to_pb.h" #include "olap/data_dir.h" #include "olap/olap_common.h" #include "olap/options.h" @@ -170,6 +172,61 @@ class BetaRowsetTest : public testing::Test { EXPECT_EQ(Status::OK(), s); } + void init_rs_meta(RowsetMetaSharedPtr& pb1, int64_t start, int64_t end) { + std::string json_rowset_meta = R"({ + "rowset_id": 540085, + "tablet_id": 15674, + "partition_id": 10000, + "txn_id": 4045, + "tablet_schema_hash": 567997588, + "rowset_type": "BETA_ROWSET", + "rowset_state": "VISIBLE", + "start_version": 2, + "end_version": 2, + "num_rows": 3929, + "total_disk_size": 84699, + "data_disk_size": 84464, + "index_disk_size": 235, + "empty": false, + "load_id": { + "hi": -5350970832824939812, + "lo": -6717994719194512122 + }, + "creation_time": 1553765670 + })"; + + RowsetMetaPB rowset_meta_pb; + json2pb::JsonToProtoMessage(json_rowset_meta, &rowset_meta_pb); + rowset_meta_pb.set_start_version(start); + rowset_meta_pb.set_end_version(end); + rowset_meta_pb.set_creation_time(10000); + + pb1->init_from_pb(rowset_meta_pb); + } + + void construct_column(ColumnPB* column_pb, TabletIndexPB* tablet_index, int64_t index_id, + const std::string& index_name, int32_t col_unique_id, + const std::string& column_type, const std::string& column_name, + const std::map& properties = + std::map(), + bool is_key = false) { + column_pb->set_unique_id(col_unique_id); + column_pb->set_name(column_name); + column_pb->set_type(column_type); + column_pb->set_is_key(is_key); + column_pb->set_is_nullable(true); + tablet_index->set_index_id(index_id); + tablet_index->set_index_name(index_name); + tablet_index->set_index_type(IndexType::INVERTED); + tablet_index->add_col_unique_id(col_unique_id); + if (!properties.empty()) { + auto* pros = tablet_index->mutable_properties(); + for (const auto& [key, value] : properties) { + (*pros)[key] = value; + } + } + } + private: std::unique_ptr _data_dir; }; @@ -304,4 +361,54 @@ TEST_F(BetaRowsetTest, AddToBinlogTest) { ASSERT_TRUE(s.ok()) << "second add_to_binlog(): " << s; } +TEST_F(BetaRowsetTest, GetIndexFileNames) { + // v1 + { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000, "key_index", 0, + "INT", "key"); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1, + "STRING", "v1"); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + auto rowset_meta = std::make_shared(); + init_rs_meta(rowset_meta, 1, 1); + rowset_meta->set_num_segments(2); + + BetaRowset rowset(tablet_schema, rowset_meta, ""); + auto file_names = rowset.get_index_file_names(); + ASSERT_EQ(file_names[0], "540085_0_10000.idx"); + ASSERT_EQ(file_names[1], "540085_0_10001.idx"); + ASSERT_EQ(file_names[2], "540085_1_10000.idx"); + ASSERT_EQ(file_names[3], "540085_1_10001.idx"); + } + + // v2 + { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000, "key_index", 0, + "INT", "key"); + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1, + "STRING", "v1"); + schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + auto tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + auto rowset_meta = std::make_shared(); + init_rs_meta(rowset_meta, 1, 1); + rowset_meta->set_num_segments(2); + + BetaRowset rowset(tablet_schema, rowset_meta, ""); + auto file_names = rowset.get_index_file_names(); + ASSERT_EQ(file_names[0], "540085_0.idx"); + ASSERT_EQ(file_names[1], "540085_1.idx"); + } +} + } // namespace doris diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 8951e420dcc6e4..7c228975f0afb6 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -3040,7 +3040,7 @@ class Suite implements GroovyInterceptable { } } - def getRowsetFileCacheDirFromBe = { beHttpPort, msHttpPort, tabletId, version -> + def getRowsetFileCacheDirFromBe = { beHttpPort, msHttpPort, tabletId, version, fileSuffix = "dat" -> def hashValues = [] def segmentFiles = [] getSegmentFilesFromMs(msHttpPort, tabletId, version) { @@ -3050,7 +3050,7 @@ class Suite implements GroovyInterceptable { // {"rowset_id":"0","partition_id":"27695","tablet_id":"27700","txn_id":"7057526525952","tablet_schema_hash":0,"rowset_type":"BETA_ROWSET","rowset_state":"COMMITTED","start_version":"3","end_version":"3","version_hash":"0","num_rows":"1","total_disk_size":"895","data_disk_size":"895","index_disk_size":"0","empty":false,"load_id":{"hi":"-1646598626735601581","lo":"-6677682539881484579"},"delete_flag":false,"creation_time":"1736153402","num_segments":"1","rowset_id_v2":"0200000000000004694889e84c76391cfd52ec7db0a483ba","resource_id":"1","newest_write_timestamp":"1736153402","segments_key_bounds":[{"min_key":"AoAAAAAAAAAC","max_key":"AoAAAAAAAAAC"}],"txn_expiration":"1736167802","segments_overlap_pb":"NONOVERLAPPING","compaction_level":"0","segments_file_size":["895"],"index_id":"27697","schema_version":0,"enable_segments_file_size":true,"has_variant_type_in_schema":false,"enable_inverted_index_file_info":false} def segmentNum = json.num_segments as int def rowsetId = json.rowset_id_v2 as String - segmentFiles = (0.. "${rowsetId}_${i}.dat" } + segmentFiles = (0.. "${rowsetId}_${i}.${fileSuffix}" } } segmentFiles.each { @@ -3064,7 +3064,7 @@ class Suite implements GroovyInterceptable { } // get table's tablet file cache - def getTabletFileCacheDirFromBe = { msHttpPort, table, version -> + def getTabletFileCacheDirFromBe = { msHttpPort, table, version, fileSuffix = "dat" -> // beHost HashFile def beHostToHashFile = [:] @@ -3072,7 +3072,7 @@ class Suite implements GroovyInterceptable { getTabletsAndHostFromFe.each { def beHost = it.Value[1] def tabletId = it.Key - def hashRet = getRowsetFileCacheDirFromBe(beHost + ":8040", msHttpPort, tabletId, version) + def hashRet = getRowsetFileCacheDirFromBe(beHost + ":8040", msHttpPort, tabletId, version, fileSuffix) hashRet.each { def hashFile = it if (beHostToHashFile.containsKey(beHost)) { diff --git a/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy b/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy index 8d41939981a812..807e51ae95ff4c 100644 --- a/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy +++ b/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_file_cache.groovy @@ -66,7 +66,7 @@ suite('test_clean_stale_rs_file_cache', 'docker') { sql """ insert into $table values (10, 1, 'v1'), (20, 2, 'v2'), (30, 3, 'v3') """ - def cacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 2) + def cacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 3) // version 4 sql """ insert into $table values (100, 1, 'v1'), (200, 2, 'v2'), (300, 3, 'v3') diff --git a/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_index_file_cache.groovy b/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_index_file_cache.groovy new file mode 100644 index 00000000000000..9077364d577a88 --- /dev/null +++ b/regression-test/suites/cloud_p0/tablets/test_clean_stale_rs_index_file_cache.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.doris.regression.util.Http + +suite('test_clean_stale_rs_index_file_cache', 'docker') { + if (!isCloudMode()) { + return; + } + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + 'sys_log_verbose_modules=org', + 'heartbeat_interval_second=1' + ] + options.beConfigs += [ + 'report_tablet_interval_seconds=1', + 'cumulative_compaction_min_deltas=5', + 'tablet_rowset_stale_sweep_by_size=false', + 'tablet_rowset_stale_sweep_time_sec=60', + 'vacuum_stale_rowsets_interval_s=10' + ] + options.setFeNum(1) + options.setBeNum(1) + options.cloudMode = true + + + def table = "test_clean_stale_rs_index_file_cache" + sql """ drop table if exists $table; """ + + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + sql """CREATE TABLE $table ( + `k1` int(11) NULL, + `k2` int(11) NULL, + `v1` varchar(2048), + INDEX v1_idx (`v1`) USING INVERTED PROPERTIES("parser" = "english", "support_phrase" = "true") COMMENT '' + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ); + """ + // version 2 + sql """ + insert into $table values (1, 1, 'v1'), (2, 2, 'v2'), (3, 3, 'v3') + """ + def cacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, table, 2, "idx") + // version 3 + sql """ + insert into $table values (10, 1, 'v1'), (20, 2, 'v2'), (30, 3, 'v3') + """ + def cacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 3, "idx") + // version 4 + sql """ + insert into $table values (100, 1, 'v1'), (200, 2, 'v2'), (300, 3, 'v3') + """ + // version 5 + sql """ + insert into $table values (1000, 1, 'v1'), (2000, 2, 'v2'), (3000, 3, 'v3') + """ + // version 6 + sql """ + insert into $table values (10000, 1, 'v1'), (20000, 2, 'v2'), (30000, 3, 'v3') + """ + + def mergedCacheDir = cacheDirVersion2 + cacheDirVersion3.collectEntries { host, hashFiles -> + [(host): cacheDirVersion2[host] ? (cacheDirVersion2[host] + hashFiles) : hashFiles] + } + for (int i = 0; i < 5; i++) { + sql """ + select count(*) from $table + """ + } + def beforeGetFromFe = getTabletAndBeHostFromFe(table) + logger.info("fe tablets {}, cache dir {}", beforeGetFromFe , mergedCacheDir) + // wait compaction finish, and vacuum_stale_rowsets work + sleep(80 * 1000) + + // check cache file has been deleted + beforeGetFromFe.each { + def tabletId = it.Key + def backendId = it.Value[0] + def backendHost = it.Value[1] + def be = cluster.getBeByBackendId(backendId.toLong()) + def dataPath = new File("${be.path}/storage/file_cache") + def subDirs = [] + + def collectDirs + collectDirs = { File dir -> + if (dir.exists()) { + dir.eachDir { subDir -> + subDirs << subDir.name + collectDirs(subDir) + } + } + } + + + collectDirs(dataPath) + logger.info("BE {} file_cache subdirs: {}", backendHost, subDirs) + def cacheDir = mergedCacheDir[backendHost] + + // add check + cacheDir.each { hashFile -> + assertFalse(subDirs.any { subDir -> subDir.startsWith(hashFile) }, + "Found unexpected cache file pattern ${hashFile} in BE ${backendHost}'s file_cache directory. " + + "Matching subdir found in: ${subDirs}") + } + } + } +}