diff --git a/dbms/src/Storages/Page/V3/BlobStore.cpp b/dbms/src/Storages/Page/V3/BlobStore.cpp index 3bd0bd9c4fa..dc527b8fc65 100644 --- a/dbms/src/Storages/Page/V3/BlobStore.cpp +++ b/dbms/src/Storages/Page/V3/BlobStore.cpp @@ -1068,26 +1068,10 @@ PageEntriesEdit BlobStore::gc(std::map & } LOG_FMT_INFO(log, "BlobStore gc will migrate {:.2f}MB into new Blobs", (1.0 * total_page_size / DB::MB)); - const auto config_file_limit = config.file_limit_size.get(); - auto alloc_size = total_page_size > config_file_limit ? config_file_limit : total_page_size; - BlobFileOffset remaining_page_size = total_page_size - alloc_size; - - // We could make the memory consumption smooth during GC. - char * data_buf = static_cast(alloc(alloc_size)); - SCOPE_EXIT({ - free(data_buf, alloc_size); - }); - - char * data_pos = data_buf; - BlobFileOffset offset_in_data = 0; - BlobFileId blobfile_id; - BlobFileOffset file_offset_beg; - std::tie(blobfile_id, file_offset_beg) = getPosFromStats(alloc_size); - auto write_blob = [this, total_page_size, &written_blobs, &write_limiter](const BlobFileId & file_id, - char * data_beg, + char * data_begin, const BlobFileOffset & file_offset, - const BlobFileOffset & data_size) { + const PageSize & data_size) { try { auto blob_file = getBlobFile(file_id); @@ -1101,7 +1085,7 @@ PageEntriesEdit BlobStore::gc(std::map & file_offset, data_size, total_page_size); - blob_file->write(data_beg, file_offset, data_size, write_limiter, /*background*/ true); + blob_file->write(data_begin, file_offset, data_size, write_limiter, /*background*/ true); } catch (DB::Exception & e) { @@ -1120,6 +1104,23 @@ PageEntriesEdit BlobStore::gc(std::map & } }; + const auto config_file_limit = config.file_limit_size.get(); + // If `total_page_size` is greater than `config_file_limit`, we need to write the page data into multiple `BlobFile`s to + // make the memory consumption smooth during GC. + auto alloc_size = total_page_size > config_file_limit ? config_file_limit : total_page_size; + BlobFileOffset remaining_page_size = total_page_size - alloc_size; + + char * data_buf = static_cast(alloc(alloc_size)); + SCOPE_EXIT({ + free(data_buf, alloc_size); + }); + + char * data_pos = data_buf; + BlobFileOffset offset_in_data = 0; + BlobFileId blobfile_id; + BlobFileOffset file_offset_begin; + std::tie(blobfile_id, file_offset_begin) = getPosFromStats(alloc_size); + // blob_file_0, [, // , // , ... ] @@ -1129,11 +1130,16 @@ PageEntriesEdit BlobStore::gc(std::map & { for (const auto & [page_id, versioned, entry] : versioned_pageid_entry_list) { - // When we can't load the remaining data. - // we will use the original buffer to find an area to load the remaining data - if (offset_in_data + entry.size > config_file_limit) + /// If `total_page_size` is greater than `config_file_limit`, we need to write the page data into multiple `BlobFile`s. + /// So there may be some page entry that cannot be fit into the current blob file, and we need to write it into the next one. + /// And we need perform the following steps before writing data into the current blob file: + /// 1. reclaim unneeded space allocated from current blob stat if `offset_in_data` < `alloc_size`; + /// 2. update `remaining_page_size`; + /// After writing data into the current blob file, we reuse the original buffer for future write. + if (offset_in_data + entry.size > alloc_size) { - assert(file_offset_beg == 0); + assert(alloc_size == config_file_limit); + assert(file_offset_begin == 0); // Remove the span that is not actually used if (offset_in_data != alloc_size) { @@ -1142,7 +1148,7 @@ PageEntriesEdit BlobStore::gc(std::map & remaining_page_size += alloc_size - offset_in_data; // Write data into Blob. - write_blob(blobfile_id, data_buf, file_offset_beg, offset_in_data); + write_blob(blobfile_id, data_buf, file_offset_begin, offset_in_data); // Reset the position to reuse the buffer allocated data_pos = data_buf; @@ -1151,7 +1157,7 @@ PageEntriesEdit BlobStore::gc(std::map & // Acquire a span from stats for remaining data auto next_alloc_size = (remaining_page_size > config_file_limit ? config_file_limit : remaining_page_size); remaining_page_size -= next_alloc_size; - std::tie(blobfile_id, file_offset_beg) = getPosFromStats(next_alloc_size); + std::tie(blobfile_id, file_offset_begin) = getPosFromStats(next_alloc_size); } // Read the data into buffer by old entry @@ -1161,7 +1167,7 @@ PageEntriesEdit BlobStore::gc(std::map & // need to be updated. PageEntryV3 new_entry = entry; new_entry.file_id = blobfile_id; - new_entry.offset = file_offset_beg + offset_in_data; + new_entry.offset = file_offset_begin + offset_in_data; new_entry.padded_size = 0; // reset padded size to be zero offset_in_data += new_entry.size; @@ -1171,9 +1177,10 @@ PageEntriesEdit BlobStore::gc(std::map & } } + // write remaining data in `data_buf` into BlobFile if (offset_in_data != 0) { - write_blob(blobfile_id, data_buf, file_offset_beg, offset_in_data); + write_blob(blobfile_id, data_buf, file_offset_begin, offset_in_data); } return edit; diff --git a/dbms/src/Storages/Page/V3/LogFile/LogFilename.h b/dbms/src/Storages/Page/V3/LogFile/LogFilename.h index 6c4c3621caf..774029353bc 100644 --- a/dbms/src/Storages/Page/V3/LogFile/LogFilename.h +++ b/dbms/src/Storages/Page/V3/LogFile/LogFilename.h @@ -39,21 +39,21 @@ struct LogFilename static LogFilename parseFrom(const String & parent_path, const String & filename, LoggerPtr log); - inline String filename(LogFileStage stage) const + inline String filename(LogFileStage file_stage) const { - assert(stage != LogFileStage::Invalid); + assert(file_stage != LogFileStage::Invalid); return fmt::format( "{}_{}_{}", - ((stage == LogFileStage::Temporary) ? LOG_FILE_PREFIX_TEMP : LOG_FILE_PREFIX_NORMAL), + ((file_stage == LogFileStage::Temporary) ? LOG_FILE_PREFIX_TEMP : LOG_FILE_PREFIX_NORMAL), log_num, level_num); } - inline String fullname(LogFileStage stage) const + inline String fullname(LogFileStage file_stage) const { - assert(stage != LogFileStage::Invalid); + assert(file_stage != LogFileStage::Invalid); assert(!parent_path.empty()); - return fmt::format("{}/{}", parent_path, filename(stage)); + return fmt::format("{}/{}", parent_path, filename(file_stage)); } }; diff --git a/dbms/src/Storages/Page/V3/PageDirectory.cpp b/dbms/src/Storages/Page/V3/PageDirectory.cpp index 951da42de1c..e1977de0f6e 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectory.cpp @@ -103,7 +103,7 @@ void VersionedPageEntries::createNewEntry(const PageVersion & ver, const PageEnt ErrorCodes::LOGICAL_ERROR); } // create a new version that inherit the `being_ref_count` of the last entry - entries.emplace(ver, EntryOrDelete::newRepalcingEntry(last_iter->second, entry)); + entries.emplace(ver, EntryOrDelete::newReplacingEntry(last_iter->second, entry)); } return; } @@ -152,7 +152,7 @@ std::shared_ptr VersionedPageEntries::createNewExternal(const } else { - // apply a external with smaller ver than delete_ver, just ignore + // apply an external with smaller ver than delete_ver, just ignore return nullptr; } } @@ -479,7 +479,8 @@ bool VersionedPageEntries::cleanOutdatedEntries( UInt64 lowest_seq, std::map> * normal_entries_to_deref, PageEntriesV3 * entries_removed, - const PageLock & /*page_lock*/) + const PageLock & /*page_lock*/, + bool keep_last_valid_var_entry) { if (type == EditRecordType::VAR_EXTERNAL) { @@ -525,8 +526,14 @@ bool VersionedPageEntries::cleanOutdatedEntries( } // If the first version less than is entry / external, - // then we can remove those entries prev of it - bool keep_if_being_ref = !iter->second.isEntry(); + // then we can remove those entries prev of it. + // If the first version less than is delete, + // we may keep the first valid entry before the delete entry in the following case: + // 1) if `keep_last_valid_var_entry` is true + // (this is only used when dump snapshot because there may be some upsert entry in later wal files, + // so we need keep the last valid entry here to avoid the delete entry being removed) + // 2) if `being_ref_count` > 1(this means the entry is ref by other entries) + bool last_entry_is_delete = !iter->second.isEntry(); --iter; // keep the first version less than while (true) { @@ -537,9 +544,9 @@ bool VersionedPageEntries::cleanOutdatedEntries( } else if (iter->second.isEntry()) { - if (keep_if_being_ref) + if (last_entry_is_delete) { - if (iter->second.being_ref_count == 1) + if (!keep_last_valid_var_entry && iter->second.being_ref_count == 1) { if (entries_removed) { @@ -549,7 +556,7 @@ bool VersionedPageEntries::cleanOutdatedEntries( } // The `being_ref_count` for this version is valid. While for older versions, // theirs `being_ref_count` is invalid, don't need to be kept - keep_if_being_ref = false; + last_entry_is_delete = false; } else { @@ -570,7 +577,7 @@ bool VersionedPageEntries::cleanOutdatedEntries( return entries.empty() || (entries.size() == 1 && entries.begin()->second.isDelete()); } -bool VersionedPageEntries::derefAndClean(UInt64 lowest_seq, PageIdV3Internal page_id, const PageVersion & deref_ver, const Int64 deref_count, PageEntriesV3 * entries_removed) +bool VersionedPageEntries::derefAndClean(UInt64 lowest_seq, PageIdV3Internal page_id, const PageVersion & deref_ver, const Int64 deref_count, PageEntriesV3 * entries_removed, bool keep_last_valid_var_entry) { auto page_lock = acquireLock(); if (type == EditRecordType::VAR_EXTERNAL) @@ -607,7 +614,7 @@ bool VersionedPageEntries::derefAndClean(UInt64 lowest_seq, PageIdV3Internal pag // Clean outdated entries after decreased the ref-counter // set `normal_entries_to_deref` to be nullptr to ignore cleaning ref-var-entries - return cleanOutdatedEntries(lowest_seq, /*normal_entries_to_deref*/ nullptr, entries_removed, page_lock); + return cleanOutdatedEntries(lowest_seq, /*normal_entries_to_deref*/ nullptr, entries_removed, page_lock, keep_last_valid_var_entry); } throw Exception(fmt::format("calling derefAndClean with invalid state [state={}]", toDebugString())); @@ -665,14 +672,22 @@ void VersionedPageEntries::collapseTo(const UInt64 seq, const PageIdV3Internal p } auto last_version = last_iter->first; auto prev_iter = --last_iter; // Note that `last_iter` should not be used anymore - if (prev_iter->second.isEntry()) + while (true) { - if (prev_iter->second.being_ref_count == 1) - return; - // It is being ref by another id, should persist the item and delete - const auto & entry = prev_iter->second; - edit.varEntry(page_id, prev_iter->first, entry.entry, entry.being_ref_count); - edit.varDel(page_id, last_version); + // if there is any entry prev to this delete entry, + // 1) the entry may be ref by another id. + // 2) the entry may be upsert into a newer wal file by the gc process. + // So we need to keep the entry item and its delete entry in the snapshot. + if (prev_iter->second.isEntry()) + { + const auto & entry = prev_iter->second; + edit.varEntry(page_id, prev_iter->first, entry.entry, entry.being_ref_count); + edit.varDel(page_id, last_version); + break; + } + if (prev_iter == entries.begin()) + break; + prev_iter--; } } return; @@ -697,7 +712,6 @@ PageDirectory::PageDirectory(String storage_name, WALStorePtr && wal_, UInt64 ma , wal(std::move(wal_)) , max_persisted_log_files(max_persisted_log_files_) , log(Logger::get("PageDirectory", std::move(storage_name))) - { } @@ -975,7 +989,7 @@ void PageDirectory::applyRefEditRecord( const PageVersion & version) { // applying ref 3->2, existing ref 2->1, normal entry 1, then we should collapse - // the ref to be 3->1, increase the refcounting of normale entry 1 + // the ref to be 3->1, increase the refcounting of normal entry 1 auto [resolve_success, resolved_id, resolved_ver] = [&mvcc_table_directory](PageIdV3Internal id_to_resolve, PageVersion ver_to_resolve) -> std::tuple { while (true) @@ -1216,12 +1230,12 @@ PageDirectory::getEntriesByBlobIds(const std::vector & blob_ids) con return std::make_pair(std::move(blob_versioned_entries), total_page_size); } -bool PageDirectory::tryDumpSnapshot(const ReadLimiterPtr & read_limiter, const WriteLimiterPtr & write_limiter) +bool PageDirectory::tryDumpSnapshot(const ReadLimiterPtr & read_limiter, const WriteLimiterPtr & write_limiter, bool force) { bool done_any_io = false; // In order not to make read amplification too high, only apply compact logs when ... auto files_snap = wal->getFilesSnapshot(); - if (files_snap.needSave(max_persisted_log_files)) + if (files_snap.needSave(max_persisted_log_files) || (force && (!files_snap.persisted_log_files.empty()))) { // To prevent writes from affecting dumping snapshot (and vice versa), old log files // are read from disk and a temporary PageDirectory is generated for dumping snapshot. @@ -1237,7 +1251,8 @@ bool PageDirectory::tryDumpSnapshot(const ReadLimiterPtr & read_limiter, const W PageDirectoryPtr collapsed_dir = factory.createFromReader( identifier, std::move(snapshot_reader), - /*wal=*/nullptr); + /* wal */ nullptr, + /* for_dump_snapshot */ true); // The records persisted in `files_snap` is older than or equal to all records in `edit` auto edit_from_disk = collapsed_dir->dumpSnapshotToEdit(); done_any_io = wal->saveSnapshot(std::move(files_snap), std::move(edit_from_disk), write_limiter); @@ -1245,7 +1260,7 @@ bool PageDirectory::tryDumpSnapshot(const ReadLimiterPtr & read_limiter, const W return done_any_io; } -PageEntriesV3 PageDirectory::gcInMemEntries(bool return_removed_entries) +PageEntriesV3 PageDirectory::gcInMemEntries(bool return_removed_entries, bool keep_last_valid_var_entry) { UInt64 lowest_seq = sequence.load(); @@ -1310,7 +1325,8 @@ PageEntriesV3 PageDirectory::gcInMemEntries(bool return_removed_entries) lowest_seq, &normal_entries_to_deref, return_removed_entries ? &all_del_entries : nullptr, - iter->second->acquireLock()); + iter->second->acquireLock(), + keep_last_valid_var_entry); { std::unique_lock write_lock(table_rw_mutex); @@ -1348,7 +1364,8 @@ PageEntriesV3 PageDirectory::gcInMemEntries(bool return_removed_entries) page_id, /*deref_ver=*/deref_counter.first, /*deref_count=*/deref_counter.second, - return_removed_entries ? &all_del_entries : nullptr); + return_removed_entries ? &all_del_entries : nullptr, + keep_last_valid_var_entry); if (all_deleted) { diff --git a/dbms/src/Storages/Page/V3/PageDirectory.h b/dbms/src/Storages/Page/V3/PageDirectory.h index 2f0f09f4e42..a1637dc8ca7 100644 --- a/dbms/src/Storages/Page/V3/PageDirectory.h +++ b/dbms/src/Storages/Page/V3/PageDirectory.h @@ -98,7 +98,7 @@ struct EntryOrDelete .entry = entry, }; } - static EntryOrDelete newRepalcingEntry(const EntryOrDelete & ori_entry, const PageEntryV3 & entry) + static EntryOrDelete newReplacingEntry(const EntryOrDelete & ori_entry, const PageEntryV3 & entry) { return EntryOrDelete{ .is_delete = false, @@ -224,13 +224,15 @@ class VersionedPageEntries UInt64 lowest_seq, std::map> * normal_entries_to_deref, PageEntriesV3 * entries_removed, - const PageLock & page_lock); + const PageLock & page_lock, + bool keep_last_valid_var_entry = false); bool derefAndClean( UInt64 lowest_seq, PageIdV3Internal page_id, const PageVersion & deref_ver, Int64 deref_count, - PageEntriesV3 * entries_removed); + PageEntriesV3 * entries_removed, + bool keep_last_valid_var_entry = false); void collapseTo(UInt64 seq, PageIdV3Internal page_id, PageEntriesEdit & edit); @@ -358,11 +360,15 @@ class PageDirectory void gcApply(PageEntriesEdit && migrated_edit, const WriteLimiterPtr & write_limiter = nullptr); - bool tryDumpSnapshot(const ReadLimiterPtr & read_limiter = nullptr, const WriteLimiterPtr & write_limiter = nullptr); + /// When create PageDirectory for dump snapshot, we should keep the last valid var_entry when it is deleted. + /// Because there may be some upsert entry in later wal files, and we should keep the valid var_entry and the delete entry to delete the later upsert entry. + /// And we don't restore the entries in blob store, because this PageDirectory is just read only for its entries. + bool tryDumpSnapshot(const ReadLimiterPtr & read_limiter = nullptr, const WriteLimiterPtr & write_limiter = nullptr, bool force = false); // Perform a GC for in-memory entries and return the removed entries. // If `return_removed_entries` is false, then just return an empty set. - PageEntriesV3 gcInMemEntries(bool return_removed_entries = true); + // When dump snapshot, we need to keep the last valid entry. Check out `tryDumpSnapshot` for the reason. + PageEntriesV3 gcInMemEntries(bool return_removed_entries = true, bool keep_last_valid_var_entry = false); std::set getAliveExternalIds(NamespaceId ns_id) const; diff --git a/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp b/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp index 968049a3273..aadef4d12a9 100644 --- a/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp +++ b/dbms/src/Storages/Page/V3/PageDirectoryFactory.cpp @@ -34,7 +34,7 @@ PageDirectoryPtr PageDirectoryFactory::create(String storage_name, FileProviderP return createFromReader(storage_name, reader, std::move(wal)); } -PageDirectoryPtr PageDirectoryFactory::createFromReader(String storage_name, WALStoreReaderPtr reader, WALStorePtr wal) +PageDirectoryPtr PageDirectoryFactory::createFromReader(String storage_name, WALStoreReaderPtr reader, WALStorePtr wal, bool for_dump_snapshot) { PageDirectoryPtr dir = std::make_unique(storage_name, std::move(wal)); loadFromDisk(dir, std::move(reader)); @@ -44,11 +44,11 @@ PageDirectoryPtr PageDirectoryFactory::createFromReader(String storage_name, WAL // After restoring from the disk, we need cleanup all invalid entries in memory, or it will // try to run GC again on some entries that are already marked as invalid in BlobStore. - // It's no need to remove the expired entries in BlobStore, so skip filling removed_entries to imporve performance. - dir->gcInMemEntries(/*return_removed_entries=*/false); + // It's no need to remove the expired entries in BlobStore, so skip filling removed_entries to improve performance. + dir->gcInMemEntries(/*return_removed_entries=*/false, /* keep_last_delete_entry */ for_dump_snapshot); LOG_FMT_INFO(DB::Logger::get("PageDirectoryFactory", storage_name), "PageDirectory restored [max_page_id={}] [max_applied_ver={}]", dir->getMaxId(), dir->sequence); - if (blob_stats) + if (!for_dump_snapshot && blob_stats) { // After all entries restored to `mvcc_table_directory`, only apply // the latest entry to `blob_stats`, or we may meet error since diff --git a/dbms/src/Storages/Page/V3/PageDirectoryFactory.h b/dbms/src/Storages/Page/V3/PageDirectoryFactory.h index a922db3b497..97a9d0da8d6 100644 --- a/dbms/src/Storages/Page/V3/PageDirectoryFactory.h +++ b/dbms/src/Storages/Page/V3/PageDirectoryFactory.h @@ -48,7 +48,7 @@ class PageDirectoryFactory PageDirectoryPtr create(String storage_name, FileProviderPtr & file_provider, PSDiskDelegatorPtr & delegator, WALStore::Config config); - PageDirectoryPtr createFromReader(String storage_name, WALStoreReaderPtr reader, WALStorePtr wal); + PageDirectoryPtr createFromReader(String storage_name, WALStoreReaderPtr reader, WALStorePtr wal, bool for_dump_snapshot = false); // just for test PageDirectoryPtr createFromEdit(String storage_name, FileProviderPtr & file_provider, PSDiskDelegatorPtr & delegator, const PageEntriesEdit & edit); diff --git a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp index f9ef25cb973..829b31c7cfe 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_page_storage.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -52,7 +53,7 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic auto path = getTemporaryPath(); createIfNotExist(path); file_provider = DB::tests::TiFlashTestEnv::getContext().getFileProvider(); - auto delegator = std::make_shared(path); + delegator = std::make_shared(path); page_storage = std::make_shared("test.t", delegator, config, file_provider); page_storage->restore(); } @@ -60,7 +61,7 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic std::shared_ptr reopenWithConfig(const PageStorage::Config & config_) { auto path = getTemporaryPath(); - auto delegator = std::make_shared(path); + delegator = std::make_shared(path); auto storage = std::make_shared("test.t", delegator, config_, file_provider); storage->restore(); return storage; @@ -70,6 +71,7 @@ class PageStorageTest : public DB::base::TiFlashStorageTestBasic protected: FileProviderPtr file_provider; std::unique_ptr path_pool; + PSDiskDelegatorPtr delegator; PageStorage::Config config; std::shared_ptr page_storage; @@ -1491,5 +1493,147 @@ try } CATCH +TEST_F(PageStorageTest, DumpPageStorageSnapshot) +try +{ + { + PageStorage::Config config; + config.blob_heavy_gc_valid_rate = 1.0; /// always run full gc + config.wal_roll_size = 1 * 1024 * 1024; /// make the wal file more easy to roll + config.wal_max_persisted_log_files = 10; /// avoid checkpoint when gc + page_storage = reopenWithConfig(config); + } + + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + PageId page_id0 = 120; + { + WriteBatch batch; + batch.putPage(page_id0, 0, std::make_shared(c_buff, buf_sz), buf_sz, {}); + page_storage->write(std::move(batch)); + } + + // create a snapshot to avoid gc + auto snap = page_storage->getSnapshot(); + + { + WriteBatch batch; + batch.delPage(page_id0); + page_storage->write(std::move(batch)); + } + + auto getLogFileNum = [&]() { + auto log_files = WALStoreReader::listAllFiles(delegator, Logger::get("PageStorageTest", "")); + return log_files.size(); + }; + + // write until there are more than one wal file + while (getLogFileNum() <= 1) + { + WriteBatch batch; + PageId page_id1 = 130; + batch.putPage(page_id1, 0, std::make_shared(c_buff, buf_sz), buf_sz, {}); + page_storage->write(std::move(batch)); + } + ASSERT_ANY_THROW(page_storage->read(page_id0)); + + // write an upsert entry into the current writing log file + auto done_full_gc = page_storage->gc(); + EXPECT_TRUE(done_full_gc); + + auto done_snapshot = page_storage->page_directory->tryDumpSnapshot(nullptr, nullptr, /* force */ true); + ASSERT_TRUE(done_snapshot); + + { + PageStorage::Config config; + page_storage = reopenWithConfig(config); + } + + ASSERT_ANY_THROW(page_storage->read(page_id0)); +} +CATCH + +TEST_F(PageStorageTest, DumpPageStorageSnapshotWithRefPage) +try +{ + { + PageStorage::Config config; + config.blob_heavy_gc_valid_rate = 1.0; /// always run full gc + config.wal_roll_size = 1 * 1024 * 1024; /// make the wal file more easy to roll + config.wal_max_persisted_log_files = 10; /// avoid checkpoint when gc + page_storage = reopenWithConfig(config); + } + + const size_t buf_sz = 1024; + char c_buff[buf_sz]; + + for (size_t i = 0; i < buf_sz; ++i) + { + c_buff[i] = i % 0xff; + } + + PageId page_id0 = 120; + { + WriteBatch batch; + batch.putPage(page_id0, 0, std::make_shared(c_buff, buf_sz), buf_sz, {}); + page_storage->write(std::move(batch)); + } + PageId page_id1 = 121; + { + WriteBatch batch; + batch.putRefPage(page_id1, page_id0); + page_storage->write(std::move(batch)); + } + // create a snapshot to avoid gc + auto snap = page_storage->getSnapshot(); + + { + WriteBatch batch; + batch.delPage(page_id0); + page_storage->write(std::move(batch)); + } + { + WriteBatch batch; + batch.delPage(page_id1); + page_storage->write(std::move(batch)); + } + + auto getLogFileNum = [&]() { + auto log_files = WALStoreReader::listAllFiles(delegator, Logger::get("PageStorageTest", "")); + return log_files.size(); + }; + + // write until there are more than one wal file + while (getLogFileNum() <= 1) + { + WriteBatch batch; + PageId page_id2 = 130; + batch.putPage(page_id2, 0, std::make_shared(c_buff, buf_sz), buf_sz, {}); + page_storage->write(std::move(batch)); + } + ASSERT_ANY_THROW(page_storage->read(page_id0)); + + // write an upsert entry into the current writing log file + auto done_full_gc = page_storage->gc(); + EXPECT_TRUE(done_full_gc); + + auto done_snapshot = page_storage->page_directory->tryDumpSnapshot(nullptr, nullptr, /* force */ true); + ASSERT_TRUE(done_snapshot); + + { + PageStorage::Config config; + page_storage = reopenWithConfig(config); + } + + ASSERT_ANY_THROW(page_storage->read(page_id0)); +} +CATCH + } // namespace PS::V3::tests } // namespace DB diff --git a/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp b/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp index 0f1406f57fc..c4d3e2f7ab7 100644 --- a/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp +++ b/dbms/src/Storages/Page/V3/tests/gtest_wal_log.cpp @@ -12,15 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. -// This source code is licensed under both the GPLv2 (found in the -// COPYING file in the root directory) and Apache 2.0 License -// (found in the LICENSE.Apache file in the root directory). -// -// Copyright (c) 2011 The LevelDB Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. See the AUTHORS file for names of contributors. - #include #include #include