From f3a7cfbf71edc738ac7cd48a48e33826d1e9b8eb Mon Sep 17 00:00:00 2001 From: random-zebra Date: Sun, 11 Apr 2021 10:20:18 +0200 Subject: [PATCH 01/20] [Trivial] Remove unused variable in ProcessNewBlock --- src/validation.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/validation.cpp b/src/validation.cpp index 8f9644441d50..fef402673385 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -3364,7 +3364,6 @@ bool ProcessNewBlock(CValidationState& state, CNode* pfrom, const std::shared_pt // Preliminary checks int64_t nStartTime = GetTimeMillis(); - const Consensus::Params& consensus = Params().GetConsensus(); int newHeight = 0; { From da6a2c15cdd65f25a344dbc82504faa1effb4f1f Mon Sep 17 00:00:00 2001 From: random-zebra Date: Fri, 26 Mar 2021 17:04:40 +0100 Subject: [PATCH 02/20] [Build] CMake: Introduce evo headers --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd3710375d41..5ed3ebf4461f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -176,6 +176,7 @@ file(GLOB CONSENSUS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/consensus/*.h) file(GLOB CTAES_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/crypto/ctaes/*.h) file(GLOB ZRUST_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/rust/include/*.h) file(GLOB SAPLING_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/sapling/*.h) +file(GLOB EVO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/evo/*.h) source_group("BitcoinHeaders" FILES ${HEADERS} @@ -189,6 +190,7 @@ source_group("BitcoinHeaders" FILES ${CTAES_HEADERS} ${ZRUST_HEADERS} ${SAPLING_HEADERS} + ${EVO_HEADERS} ./src/support/cleanse.h ) From 24969335f67e41b2e108fa10a976e0637e8ef87a Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 14 Feb 2018 12:04:24 +0100 Subject: [PATCH 03/20] Implement CDBTransaction and CScopedDBTransaction >>> backports dash@4531f6b896119b2b8f33c03a50cb4d4ee62a6acb Allows easier commit/rollback handling, especially useful when AcceptBlock fails and things need to be reverted. --- src/dbwrapper.h | 214 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 4700d3b453be..cf51f6803130 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -12,6 +12,7 @@ #include "util.h" #include "version.h" +#include #include #include @@ -309,4 +310,217 @@ class CDBWrapper }; +class CDBTransaction { +private: + CDBWrapper &db; + + struct KeyHolder { + virtual ~KeyHolder() = default; + virtual bool Less(const KeyHolder &b) const = 0; + virtual void Erase(CDBBatch &batch) = 0; + }; + typedef std::unique_ptr KeyHolderPtr; + + template + struct KeyHolderImpl : KeyHolder { + KeyHolderImpl(const K &_key) + : key(_key) { + } + virtual bool Less(const KeyHolder &b) const { + auto *b2 = dynamic_cast*>(&b); + return key < b2->key; + } + virtual void Erase(CDBBatch &batch) { + batch.Erase(key); + } + K key; + }; + + struct KeyValueHolder { + virtual void Write(CDBBatch &batch) = 0; + }; + typedef std::unique_ptr KeyValueHolderPtr; + + template + struct KeyValueHolderImpl : KeyValueHolder { + KeyValueHolderImpl(const KeyHolderImpl &_key, const V &_value) + : key(_key), + value(_value) { } + virtual void Write(CDBBatch &batch) { + batch.Write(key.key, value); + } + const KeyHolderImpl &key; + V value; + }; + + struct keyCmp { + bool operator()(const KeyHolderPtr &a, const KeyHolderPtr &b) const { + return a->Less(*b); + } + }; + + typedef std::map KeyValueMap; + typedef std::map TypeKeyValueMap; + + TypeKeyValueMap writes; + TypeKeyValueMap deletes; + + template + KeyValueMap *getMapForType(TypeKeyValueMap &m, bool create) { + auto it = m.find(typeid(K)); + if (it != m.end()) { + return &it->second; + } + if (!create) + return nullptr; + auto it2 = m.emplace(typeid(K), KeyValueMap()); + return &it2.first->second; + } + + template + KeyValueMap *getWritesMap(bool create) { + return getMapForType(writes, create); + } + + template + KeyValueMap *getDeletesMap(bool create) { + return getMapForType(deletes, create); + } + +public: + CDBTransaction(CDBWrapper &_db) : db(_db) {} + + template + void Write(const K& key, const V& value) { + KeyHolderPtr k(new KeyHolderImpl(key)); + KeyHolderImpl* k2 = dynamic_cast*>(k.get()); + KeyValueHolderPtr kv(new KeyValueHolderImpl(*k2, value)); + + KeyValueMap *ds = getDeletesMap(false); + if (ds) + ds->erase(k); + + KeyValueMap *ws = getWritesMap(true); + ws->erase(k); + ws->emplace(std::make_pair(std::move(k), std::move(kv))); + } + + template + bool Read(const K& key, V& value) { + KeyHolderPtr k(new KeyHolderImpl(key)); + + KeyValueMap *ds = getDeletesMap(false); + if (ds && ds->count(k)) + return false; + + KeyValueMap *ws = getWritesMap(false); + if (ws) { + KeyValueMap::iterator it = ws->find(k); + if (it != ws->end()) { + auto *impl = dynamic_cast *>(it->second.get()); + if (!impl) + return false; + value = impl->value; + return true; + } + } + + return db.Read(key, value); + } + + template + bool Exists(const K& key) { + KeyHolderPtr k(new KeyHolderImpl(key)); + + KeyValueMap *ds = getDeletesMap(false); + if (ds && ds->count(k)) + return false; + + KeyValueMap *ws = getWritesMap(false); + if (ws && ws->count(k)) + return true; + + return db.Exists(key); + } + + template + void Erase(const K& key) { + KeyHolderPtr k(new KeyHolderImpl(key)); + + KeyValueMap *ws = getWritesMap(false); + if (ws) + ws->erase(k); + KeyValueMap *ds = getDeletesMap(true); + ds->emplace(std::move(k), nullptr); + } + + void Clear() { + writes.clear(); + deletes.clear(); + } + + bool Commit() { + CDBBatch batch; + for (auto &p : deletes) { + for (auto &p2 : p.second) { + p2.first->Erase(batch); + } + } + for (auto &p : writes) { + for (auto &p2 : p.second) { + p2.second->Write(batch); + } + } + bool ret = db.WriteBatch(batch, true); + Clear(); + return ret; + } + + bool IsClean() { + return writes.empty() && deletes.empty(); + } +}; + +class CScopedDBTransaction { +private: + CDBTransaction &dbTransaction; + std::function commitHandler; + std::function rollbackHandler; + bool didCommitOrRollback{}; + +public: + CScopedDBTransaction(CDBTransaction &dbTx) : dbTransaction(dbTx) {} + ~CScopedDBTransaction() { + if (!didCommitOrRollback) + Rollback(); + } + bool Commit() { + assert(!didCommitOrRollback); + didCommitOrRollback = true; + bool result = dbTransaction.Commit(); + if (commitHandler) + commitHandler(); + return result; + } + void Rollback() { + assert(!didCommitOrRollback); + didCommitOrRollback = true; + dbTransaction.Clear(); + if (rollbackHandler) + rollbackHandler(); + } + + static std::unique_ptr Begin(CDBTransaction &dbTx) { + assert(dbTx.IsClean()); + return std::unique_ptr(new CScopedDBTransaction(dbTx)); + } + + void SetCommitHandler(const std::function &h) { + commitHandler = h; + } + void SetRollbackHandler(const std::function &h) { + rollbackHandler = h; + } +}; + #endif // BITCOIN_DBWRAPPER_H From d6c56d17de370ffa1c184d5b53c9d2423d7be39a Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 24 May 2018 14:28:23 +0200 Subject: [PATCH 04/20] Introduce CEvoDB for all evo related things, e.g. DIP3 >>> backports dash@c9a72e8880fe88f07b64aecfc059f08ec14440fc Also add transaction handling to ConnectTip and DisconnectTip and a few other places where blocks are processed. --- CMakeLists.txt | 1 + src/Makefile.am | 2 ++ src/evo/evodb.cpp | 14 +++++++++ src/evo/evodb.h | 65 ++++++++++++++++++++++++++++++++++++++++++ src/init.cpp | 6 ++++ src/test/test_pivx.cpp | 3 ++ src/validation.cpp | 22 ++++++++++++-- 7 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 src/evo/evodb.cpp create mode 100644 src/evo/evodb.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ed3ebf4461f..7155f1b72962 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -385,6 +385,7 @@ set(COMMON_SOURCES ./src/coins.cpp ./src/key_io.cpp ./src/compressor.cpp + ./src/evo/evodb.cpp ./src/evo/specialtx.cpp ./src/consensus/merkle.cpp ./src/consensus/zerocoin_verify.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 59e2e67dc249..94dd80a92109 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -181,6 +181,7 @@ BITCOIN_CORE_H = \ cuckoocache.h \ crypter.h \ cyclingvector.h \ + evo/evodb.h \ evo/specialtx.h \ pairresult.h \ addressbook.h \ @@ -322,6 +323,7 @@ libbitcoin_server_a_SOURCES = \ consensus/params.cpp \ consensus/tx_verify.cpp \ consensus/zerocoin_verify.cpp \ + evo/evodb.cpp \ evo/specialtx.cpp \ httprpc.cpp \ httpserver.cpp \ diff --git a/src/evo/evodb.cpp b/src/evo/evodb.cpp new file mode 100644 index 000000000000..36b1aac5b705 --- /dev/null +++ b/src/evo/evodb.cpp @@ -0,0 +1,14 @@ +// Copyright (c) 2018-2021 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "evodb.h" + +CEvoDB* evoDb; + +CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) : + db(GetDataDir() / "evodb", nCacheSize, fMemory, fWipe), + dbTransaction(db) +{ +} diff --git a/src/evo/evodb.h b/src/evo/evodb.h new file mode 100644 index 000000000000..908815ecf86b --- /dev/null +++ b/src/evo/evodb.h @@ -0,0 +1,65 @@ +// Copyright (c) 2018-2021 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef PIVX_EVODB_H +#define PIVX_EVODB_H + +#include "dbwrapper.h" +#include "sync.h" + +class CEvoDB +{ +private: + RecursiveMutex cs; + CDBWrapper db; + CDBTransaction dbTransaction; + +public: + CEvoDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + + std::unique_ptr BeginTransaction() + { + LOCK(cs); + auto t = CScopedDBTransaction::Begin(dbTransaction); + return t; + } + + template + bool Read(const K& key, V& value) + { + LOCK(cs); + return dbTransaction.Read(key, value); + } + + template + void Write(const K& key, const V& value) + { + LOCK(cs); + dbTransaction.Write(key, value); + } + + template + bool Exists(const K& key) + { + LOCK(cs); + return dbTransaction.Exists(key); + } + + template + void Erase(const K& key) + { + LOCK(cs); + dbTransaction.Erase(key); + } + + CDBWrapper& GetRawDB() + { + return db; + } +}; + +extern CEvoDB* evoDb; + +#endif//PIVX_EVODB_H diff --git a/src/init.cpp b/src/init.cpp index d7b4cbbcf53a..a6bc0cc24c05 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -44,6 +44,7 @@ #include "scheduler.h" #include "spork.h" #include "sporkdb.h" +#include "evo/evodb.h" #include "txdb.h" #include "torcontrol.h" #include "guiinterface.h" @@ -297,6 +298,8 @@ void PrepareShutdown() zerocoinDB = NULL; delete pSporkDB; pSporkDB = NULL; + delete evoDb; + evoDb = nullptr; } #ifdef ENABLE_WALLET if (pwalletMain) @@ -1547,6 +1550,7 @@ bool AppInitMain() int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nTotalCache -= nCoinDBCache; nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache + int64_t nEvoDbCache = 1024 * 1024 * 16; // TODO LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1fMiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024)); @@ -1571,10 +1575,12 @@ bool AppInitMain() delete pblocktree; delete zerocoinDB; delete pSporkDB; + delete evoDb; //PIVX specific: zerocoin and spork DB's zerocoinDB = new CZerocoinDB(0, false, fReindex); pSporkDB = new CSporkDB(0, false, false); + evoDb = new CEvoDB(nEvoDbCache, false, fReindex); pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); diff --git a/src/test/test_pivx.cpp b/src/test/test_pivx.cpp index adad49eb9192..90b66d85d9f5 100644 --- a/src/test/test_pivx.cpp +++ b/src/test/test_pivx.cpp @@ -9,6 +9,7 @@ #include "blockassembler.h" #include "guiinterface.h" +#include "evo/evodb.h" #include "miner.h" #include "net_processing.h" #include "random.h" @@ -75,6 +76,7 @@ TestingSetup::TestingSetup() RegisterAllCoreRPCCommands(tableRPC); zerocoinDB = new CZerocoinDB(0, true); pSporkDB = new CSporkDB(0, true); + evoDb = new CEvoDB(1 << 20, true, true); pblocktree = new CBlockTreeDB(1 << 20, true); pcoinsdbview = new CCoinsViewDB(1 << 23, true); pcoinsTip = new CCoinsViewCache(pcoinsdbview); @@ -109,6 +111,7 @@ TestingSetup::~TestingSetup() delete pblocktree; delete zerocoinDB; delete pSporkDB; + delete evoDb; fs::remove_all(pathTemp); } diff --git a/src/validation.cpp b/src/validation.cpp index fef402673385..ec1833e3aca5 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -43,6 +43,7 @@ #include "script/sigcache.h" #include "spork.h" #include "sporkdb.h" +#include "evo/evodb.h" #include "txdb.h" #include "txmempool.h" #include "undo.h" @@ -1954,11 +1955,16 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara const uint256& saplingAnchorBeforeDisconnect = pcoinsTip->GetBestAnchor(); int64_t nStart = GetTimeMicros(); { + auto dbTx = evoDb->BeginTransaction(); + CCoinsViewCache view(pcoinsTip); assert(view.GetBestBlock() == pindexDelete->GetBlockHash()); if (DisconnectBlock(block, pindexDelete, view) != DISCONNECT_OK) return error("DisconnectTip() : DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); - assert(view.Flush()); + bool flushed = view.Flush(); + assert(flushed); + bool committed = dbTx->Commit(); + assert(committed); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); const uint256& saplingAnchorAfterDisconnect = pcoinsTip->GetBestAnchor(); @@ -2103,6 +2109,8 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const st int64_t nTime3; LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { + auto dbTx = evoDb->BeginTransaction(); + CCoinsViewCache view(pcoinsTip); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view, false); GetMainSignals().BlockChecked(blockConnecting, state); @@ -2114,7 +2122,10 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const st nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); - assert(view.Flush()); + bool flushed = view.Flush(); + assert(flushed); + bool committed = dbTx->Commit(); + assert(committed); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; @@ -3415,6 +3426,9 @@ bool TestBlockValidity(CValidationState& state, const CBlock& block, CBlockIndex indexDummy.pprev = pindexPrev; indexDummy.nHeight = pindexPrev->nHeight + 1; + // begin tx and let it rollback + auto dbTx = evoDb->BeginTransaction(); + // NOTE: CheckBlockHeader is called by CheckBlock if (!ContextualCheckBlockHeader(block, state, pindexPrev)) return error("%s: ContextualCheckBlockHeader failed: %s", __func__, FormatStateMessage(state)); @@ -3654,6 +3668,10 @@ bool CVerifyDB::VerifyDB(CCoinsView* coinsview, int nCheckLevel, int nCheckDepth return true; const int chainHeight = chainActive.Height(); + + // begin tx and let it rollback + auto dbTx = evoDb->BeginTransaction(); + // Verify blocks in the best chain if (nCheckDepth <= 0) nCheckDepth = 1000000000; // suffices until the year 19000 From ad7f5d7f2f4b5b942d69d412f8c86a3ab60fbef4 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Tue, 2 Mar 2021 00:31:32 +0100 Subject: [PATCH 05/20] [DB][BUG] Add virtual destructor for KeyValueHolder --- src/dbwrapper.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index cf51f6803130..1ab5ee1af80f 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -337,6 +337,7 @@ class CDBTransaction { }; struct KeyValueHolder { + virtual ~KeyValueHolder() = default; virtual void Write(CDBBatch &batch) = 0; }; typedef std::unique_ptr KeyValueHolderPtr; From cab50d31469f1f746cd8fe089cc86647ce05dd57 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Mon, 25 Jan 2021 09:58:35 +0100 Subject: [PATCH 06/20] [Tests] Fix functional test suite with new directory 'evodb' --- test/functional/test_framework/test_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/functional/test_framework/test_framework.py b/test/functional/test_framework/test_framework.py index 7bba72319539..bc5ffca1cc80 100755 --- a/test/functional/test_framework/test_framework.py +++ b/test/functional/test_framework/test_framework.py @@ -490,7 +490,7 @@ def copy_and_overwrite(from_path, to_path): node_0_datadir = os.path.join(get_datadir_path(cachedir, 0), "regtest") for i in range(from_num, MAX_NODES): node_i_datadir = os.path.join(get_datadir_path(cachedir, i), "regtest") - for subdir in ["blocks", "chainstate", "sporks", "zerocoin"]: + for subdir in ["blocks", "chainstate", "evodb", "sporks", "zerocoin"]: copy_and_overwrite(os.path.join(node_0_datadir, subdir), os.path.join(node_i_datadir, subdir)) initialize_datadir(cachedir, i) # Overwrite port/rpcport in pivx.conf @@ -510,7 +510,7 @@ def cache_path(n, *paths): for i in range(MAX_NODES): for entry in os.listdir(cache_path(i)): - if entry not in ['wallet.dat', 'chainstate', 'blocks', 'sporks', 'zerocoin', 'backups']: + if entry not in ['wallet.dat', 'chainstate', 'blocks', 'sporks', 'evodb', 'zerocoin', 'backups']: os.remove(cache_path(i, entry)) def clean_cache_dir(): From be85c9a7a61218692b15549773f49092cd9193cc Mon Sep 17 00:00:00 2001 From: random-zebra Date: Wed, 27 Jan 2021 17:24:52 +0100 Subject: [PATCH 07/20] [Refactoring] migrate evoDb to unique pointer --- src/evo/evodb.cpp | 2 +- src/evo/evodb.h | 2 +- src/init.cpp | 8 ++++---- src/test/test_pivx.cpp | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/evo/evodb.cpp b/src/evo/evodb.cpp index 36b1aac5b705..d8d2f05fdb6b 100644 --- a/src/evo/evodb.cpp +++ b/src/evo/evodb.cpp @@ -5,7 +5,7 @@ #include "evodb.h" -CEvoDB* evoDb; +std::unique_ptr evoDb; CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "evodb", nCacheSize, fMemory, fWipe), diff --git a/src/evo/evodb.h b/src/evo/evodb.h index 908815ecf86b..70ff74ef6ade 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -60,6 +60,6 @@ class CEvoDB } }; -extern CEvoDB* evoDb; +extern std::unique_ptr evoDb; #endif//PIVX_EVODB_H diff --git a/src/init.cpp b/src/init.cpp index a6bc0cc24c05..9c4b89cc64bd 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -298,8 +298,7 @@ void PrepareShutdown() zerocoinDB = NULL; delete pSporkDB; pSporkDB = NULL; - delete evoDb; - evoDb = nullptr; + evoDb.reset(); } #ifdef ENABLE_WALLET if (pwalletMain) @@ -1575,12 +1574,13 @@ bool AppInitMain() delete pblocktree; delete zerocoinDB; delete pSporkDB; - delete evoDb; //PIVX specific: zerocoin and spork DB's zerocoinDB = new CZerocoinDB(0, false, fReindex); pSporkDB = new CSporkDB(0, false, false); - evoDb = new CEvoDB(nEvoDbCache, false, fReindex); + + evoDb.reset(); + evoDb.reset(new CEvoDB(nEvoDbCache, false, fReindex)); pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex); diff --git a/src/test/test_pivx.cpp b/src/test/test_pivx.cpp index 90b66d85d9f5..05ca41899038 100644 --- a/src/test/test_pivx.cpp +++ b/src/test/test_pivx.cpp @@ -47,11 +47,13 @@ BasicTestingSetup::BasicTestingSetup() InitSignatureCache(); fCheckBlockIndex = true; SelectParams(CBaseChainParams::MAIN); + evoDb.reset(new CEvoDB(1 << 20, true, true)); } BasicTestingSetup::~BasicTestingSetup() { ECC_Stop(); g_connman.reset(); + evoDb.reset(); } TestingSetup::TestingSetup() @@ -76,7 +78,6 @@ TestingSetup::TestingSetup() RegisterAllCoreRPCCommands(tableRPC); zerocoinDB = new CZerocoinDB(0, true); pSporkDB = new CSporkDB(0, true); - evoDb = new CEvoDB(1 << 20, true, true); pblocktree = new CBlockTreeDB(1 << 20, true); pcoinsdbview = new CCoinsViewDB(1 << 23, true); pcoinsTip = new CCoinsViewCache(pcoinsdbview); @@ -111,7 +112,6 @@ TestingSetup::~TestingSetup() delete pblocktree; delete zerocoinDB; delete pSporkDB; - delete evoDb; fs::remove_all(pathTemp); } From fec56b034015f8844aeee7a67546783efb75c6c0 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 10:23:57 +0100 Subject: [PATCH 08/20] [DB] Ensure evoDB consistency by storing best block hash >>> adapted from dash@378dadd0f7c73285ab5a13d1f6fb96314822170b --- src/evo/evodb.cpp | 11 +++++++++++ src/evo/evodb.h | 6 ++++++ src/validation.cpp | 24 +++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/evo/evodb.cpp b/src/evo/evodb.cpp index d8d2f05fdb6b..0b5eefcbeb02 100644 --- a/src/evo/evodb.cpp +++ b/src/evo/evodb.cpp @@ -12,3 +12,14 @@ CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) : dbTransaction(db) { } + +bool CEvoDB::VerifyBestBlock(const uint256& hash) +{ + uint256 hashBestBlock; + return Read(EVODB_BEST_BLOCK, hashBestBlock) && hashBestBlock == hash; +} + +void CEvoDB::WriteBestBlock(const uint256& hash) +{ + Write(EVODB_BEST_BLOCK, hash); +} diff --git a/src/evo/evodb.h b/src/evo/evodb.h index 70ff74ef6ade..9afe2614cc2c 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -8,6 +8,9 @@ #include "dbwrapper.h" #include "sync.h" +#include "uint256.h" + +static const std::string EVODB_BEST_BLOCK = "b_b"; class CEvoDB { @@ -58,6 +61,9 @@ class CEvoDB { return db; } + + bool VerifyBestBlock(const uint256& hash); + void WriteBestBlock(const uint256& hash); }; extern std::unique_ptr evoDb; diff --git a/src/validation.cpp b/src/validation.cpp index ec1833e3aca5..e9bf06e9bcb1 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1313,6 +1313,15 @@ int ApplyTxInUndo(Coin&& undo, CCoinsViewCache& view, const COutPoint& out) DisconnectResult DisconnectBlock(CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) { AssertLockHeld(cs_main); + + bool fDIP3Active = Params().GetConsensus().NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_V6_0); + bool fHasBestBlock = evoDb->VerifyBestBlock(pindex->GetBlockHash()); + + if (fDIP3Active && !fHasBestBlock) { + AbortNode("Found EvoDB inconsistency, you must reindex to continue"); + return DISCONNECT_FAILED; + } + bool fClean = true; CBlockUndo blockUndo; @@ -1394,6 +1403,7 @@ DisconnectResult DisconnectBlock(CBlock& block, const CBlockIndex* pindex, CCoin // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); + evoDb->WriteBestBlock(pindex->pprev->GetBlockHash()); if (consensus.NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_ZC_V2) && pindex->nHeight <= consensus.height_last_ZC_AccumCheckpoint) { @@ -1466,13 +1476,23 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd LogPrintf("%s: hashPrev=%s view=%s\n", __func__, hashPrevBlock.GetHex(), view.GetBestBlock().GetHex()); assert(hashPrevBlock == view.GetBestBlock()); + if (pindex->pprev) { + bool fDIP3Active = Params().GetConsensus().NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_V6_0); + bool fHasBestBlock = evoDb->VerifyBestBlock(hashPrevBlock); + + if (fDIP3Active && !fHasBestBlock) { + return AbortNode(state, "Found EvoDB inconsistency, you must reindex to continue"); + } + } + const Consensus::Params& consensus = Params().GetConsensus(); // Special case for the genesis block, skipping connection of its transactions // (its coinbase is unspendable) if (block.GetHash() == consensus.hashGenesisBlock) { - if (!fJustCheck) + if (!fJustCheck) { view.SetBestBlock(pindex->GetBlockHash()); + } return true; } @@ -1757,6 +1777,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); + evoDb->WriteBestBlock(pindex->GetBlockHash()); int64_t nTime4 = GetTimeMicros(); nTimeIndex += nTime4 - nTime3; @@ -3833,6 +3854,7 @@ bool ReplayBlocks(const CChainParams& params, CCoinsView* view) } cache.SetBestBlock(pindexNew->GetBlockHash()); + evoDb->WriteBestBlock(pindexNew->GetBlockHash()); cache.Flush(); uiInterface.ShowProgress("", 100); return true; From e6c7efe14f15be64f42585dbf9ad7c1a641c5c47 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 10:34:11 +0100 Subject: [PATCH 09/20] [Refactoring] Let Commit() return void >>> Backports dash@a1cca1b9a11975b8a57d916d1325dc8d855217b7 the boolean value will lose its meaning in the next commit --- src/dbwrapper.h | 10 ++++------ src/validation.cpp | 6 ++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 1ab5ee1af80f..a116954a0277 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -460,7 +460,7 @@ class CDBTransaction { deletes.clear(); } - bool Commit() { + void Commit() { CDBBatch batch; for (auto &p : deletes) { for (auto &p2 : p.second) { @@ -472,9 +472,8 @@ class CDBTransaction { p2.second->Write(batch); } } - bool ret = db.WriteBatch(batch, true); + db.WriteBatch(batch, true); Clear(); - return ret; } bool IsClean() { @@ -495,13 +494,12 @@ class CScopedDBTransaction { if (!didCommitOrRollback) Rollback(); } - bool Commit() { + void Commit() { assert(!didCommitOrRollback); didCommitOrRollback = true; - bool result = dbTransaction.Commit(); + dbTransaction.Commit(); if (commitHandler) commitHandler(); - return result; } void Rollback() { assert(!didCommitOrRollback); diff --git a/src/validation.cpp b/src/validation.cpp index e9bf06e9bcb1..9d461b4e19d3 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1984,8 +1984,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara return error("DisconnectTip() : DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); bool flushed = view.Flush(); assert(flushed); - bool committed = dbTx->Commit(); - assert(committed); + dbTx->Commit(); } LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); const uint256& saplingAnchorAfterDisconnect = pcoinsTip->GetBestAnchor(); @@ -2145,8 +2144,7 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const st LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); - bool committed = dbTx->Commit(); - assert(committed); + dbTx->Commit(); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; From 6a3696860aecfef07264a179a8f92f43c181fc4e Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Wed, 6 Mar 2019 20:45:39 +0100 Subject: [PATCH 10/20] Implement 2-stage commit for CEvoDB to avoid inconsistencies >>> from dash@521d4ae08fe0e35c30bf05ffe8b9d611e86c3bfd CDBTransaction is changed to allow CDBBatch, CDBWrapper and other CDBTransactions as parent instead of just CDBWrapper. This in turn allows to implement multi-staged commits in CEvoDB. We now have the "current transaction" which is started and ended (commit or rollback) for each call to Connect-/DisconnectBlock. When the current transaction is committed, it moves its contents into the "root transaction" instead of directly writing to CDBWrapper. CommitRootTransaction() then handles the final commitment to CDBWrapper. It is called at the same time when the chainstate is flushed to disk, which guarantees consistency between chainstate and CEvoDB. --- src/dbwrapper.h | 81 +++++++++++++++++++++++++++++----------------- src/evo/evodb.cpp | 15 +++++++-- src/evo/evodb.h | 23 +++++++++---- src/validation.cpp | 3 ++ 4 files changed, 83 insertions(+), 39 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index a116954a0277..ef8400045a84 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -310,14 +310,16 @@ class CDBWrapper }; +template class CDBTransaction { -private: - CDBWrapper &db; +protected: + Parent &parent; + CommitTarget &commitTarget; struct KeyHolder { virtual ~KeyHolder() = default; virtual bool Less(const KeyHolder &b) const = 0; - virtual void Erase(CDBBatch &batch) = 0; + virtual void Erase(CommitTarget &commitTarget) = 0; }; typedef std::unique_ptr KeyHolderPtr; @@ -330,15 +332,15 @@ class CDBTransaction { auto *b2 = dynamic_cast*>(&b); return key < b2->key; } - virtual void Erase(CDBBatch &batch) { - batch.Erase(key); + virtual void Erase(CommitTarget &commitTarget) { + commitTarget.Erase(key); } K key; }; struct KeyValueHolder { virtual ~KeyValueHolder() = default; - virtual void Write(CDBBatch &batch) = 0; + virtual void Write(CommitTarget &parent) = 0; }; typedef std::unique_ptr KeyValueHolderPtr; @@ -347,8 +349,13 @@ class CDBTransaction { KeyValueHolderImpl(const KeyHolderImpl &_key, const V &_value) : key(_key), value(_value) { } - virtual void Write(CDBBatch &batch) { - batch.Write(key.key, value); + KeyValueHolderImpl(const KeyHolderImpl &_key, V &&_value) + : key(_key), + value(std::forward(_value)) { } + virtual void Write(CommitTarget &commitTarget) { + // we're moving the value instead of copying it. This means that Write() can only be called once per + // KeyValueHolderImpl instance. Commit() clears the write maps, so this ok. + commitTarget.Write(key.key, std::move(value)); } const KeyHolderImpl &key; V value; @@ -388,22 +395,34 @@ class CDBTransaction { return getMapForType(deletes, create); } -public: - CDBTransaction(CDBWrapper &_db) : db(_db) {} - - template - void Write(const K& key, const V& value) { - KeyHolderPtr k(new KeyHolderImpl(key)); - KeyHolderImpl* k2 = dynamic_cast*>(k.get()); - KeyValueHolderPtr kv(new KeyValueHolderImpl(*k2, value)); + template + void writeImpl(KeyHolderImpl* k, KV&& kv) { + auto k2 = KeyHolderPtr(k); KeyValueMap *ds = getDeletesMap(false); if (ds) - ds->erase(k); + ds->erase(k2); KeyValueMap *ws = getWritesMap(true); - ws->erase(k); - ws->emplace(std::make_pair(std::move(k), std::move(kv))); + ws->erase(k2); + ws->emplace(std::make_pair(std::move(k2), std::forward(kv))); + } + +public: + CDBTransaction(Parent &_parent, CommitTarget &_commitTarget) : parent(_parent), commitTarget(_commitTarget) {} + + template + void Write(const K& key, const V& v) { + auto k = new KeyHolderImpl(key); + auto kv = std::make_unique>(*k, v); + writeImpl(k, std::move(kv)); + } + + template + void Write(const K& key, V&& v) { + auto k = new KeyHolderImpl(key); + auto kv = std::make_unique::type>>(*k, std::forward(v)); + writeImpl(k, std::move(kv)); } template @@ -416,7 +435,7 @@ class CDBTransaction { KeyValueMap *ws = getWritesMap(false); if (ws) { - KeyValueMap::iterator it = ws->find(k); + auto it = ws->find(k); if (it != ws->end()) { auto *impl = dynamic_cast *>(it->second.get()); if (!impl) @@ -426,7 +445,7 @@ class CDBTransaction { } } - return db.Read(key, value); + return parent.Read(key, value); } template @@ -441,7 +460,7 @@ class CDBTransaction { if (ws && ws->count(k)) return true; - return db.Exists(key); + return parent.Exists(key); } template @@ -461,18 +480,16 @@ class CDBTransaction { } void Commit() { - CDBBatch batch; for (auto &p : deletes) { for (auto &p2 : p.second) { - p2.first->Erase(batch); + p2.first->Erase(commitTarget); } } for (auto &p : writes) { for (auto &p2 : p.second) { - p2.second->Write(batch); + p2.second->Write(commitTarget); } } - db.WriteBatch(batch, true); Clear(); } @@ -481,15 +498,19 @@ class CDBTransaction { } }; +template class CScopedDBTransaction { +public: + typedef CDBTransaction Transaction; + private: - CDBTransaction &dbTransaction; + Transaction &dbTransaction; std::function commitHandler; std::function rollbackHandler; bool didCommitOrRollback{}; public: - CScopedDBTransaction(CDBTransaction &dbTx) : dbTransaction(dbTx) {} + CScopedDBTransaction(Transaction &dbTx) : dbTransaction(dbTx) {} ~CScopedDBTransaction() { if (!didCommitOrRollback) Rollback(); @@ -509,9 +530,9 @@ class CScopedDBTransaction { rollbackHandler(); } - static std::unique_ptr Begin(CDBTransaction &dbTx) { + static std::unique_ptr> Begin(Transaction &dbTx) { assert(dbTx.IsClean()); - return std::unique_ptr(new CScopedDBTransaction(dbTx)); + return std::make_unique>(dbTx); } void SetCommitHandler(const std::function &h) { diff --git a/src/evo/evodb.cpp b/src/evo/evodb.cpp index 0b5eefcbeb02..29e3abb9cd9f 100644 --- a/src/evo/evodb.cpp +++ b/src/evo/evodb.cpp @@ -8,11 +8,22 @@ std::unique_ptr evoDb; CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) : - db(GetDataDir() / "evodb", nCacheSize, fMemory, fWipe), - dbTransaction(db) + db(fMemory ? "" : (GetDataDir() / "evodb"), nCacheSize, fMemory, fWipe), + rootBatch(), + rootDBTransaction(db, rootBatch), + curDBTransaction(rootDBTransaction, rootDBTransaction) { } +bool CEvoDB::CommitRootTransaction() +{ + assert(curDBTransaction.IsClean()); + rootDBTransaction.Commit(); + bool ret = db.WriteBatch(rootBatch); + rootBatch.Clear(); + return ret; +} + bool CEvoDB::VerifyBestBlock(const uint256& hash) { uint256 hashBestBlock; diff --git a/src/evo/evodb.h b/src/evo/evodb.h index 9afe2614cc2c..f372ac2a61ec 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -17,15 +17,22 @@ class CEvoDB private: RecursiveMutex cs; CDBWrapper db; - CDBTransaction dbTransaction; + + typedef CDBTransaction RootTransaction; + typedef CDBTransaction CurTransaction; + typedef CScopedDBTransaction ScopedTransaction; + + CDBBatch rootBatch; + RootTransaction rootDBTransaction; + CurTransaction curDBTransaction; public: CEvoDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - std::unique_ptr BeginTransaction() + std::unique_ptr BeginTransaction() { LOCK(cs); - auto t = CScopedDBTransaction::Begin(dbTransaction); + auto t = ScopedTransaction::Begin(curDBTransaction); return t; } @@ -33,28 +40,28 @@ class CEvoDB bool Read(const K& key, V& value) { LOCK(cs); - return dbTransaction.Read(key, value); + return curDBTransaction.Read(key, value); } template void Write(const K& key, const V& value) { LOCK(cs); - dbTransaction.Write(key, value); + curDBTransaction.Write(key, value); } template bool Exists(const K& key) { LOCK(cs); - return dbTransaction.Exists(key); + return curDBTransaction.Exists(key); } template void Erase(const K& key) { LOCK(cs); - dbTransaction.Erase(key); + curDBTransaction.Erase(key); } CDBWrapper& GetRawDB() @@ -62,6 +69,8 @@ class CEvoDB return db; } + bool CommitRootTransaction(); + bool VerifyBestBlock(const uint256& hash); void WriteBestBlock(const uint256& hash); }; diff --git a/src/validation.cpp b/src/validation.cpp index 9d461b4e19d3..b9a156950c35 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1881,6 +1881,9 @@ bool static FlushStateToDisk(CValidationState& state, FlushStateMode mode) // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return AbortNode(state, "Failed to write to coin database"); + if (!evoDb->CommitRootTransaction()) { + return AbortNode(state, "Failed to commit EvoDB"); + } nLastFlush = nNow; // Update money supply on memory, reading data from disk if (!ShutdownRequested() && !IsInitialBlockDownload()) { From 34a7541298b53af69435cbf1155ee56bba918dcb Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 11:01:29 +0100 Subject: [PATCH 11/20] [Refactoring] Properly add explicit specifier where appropriate --- src/dbwrapper.h | 2 +- src/evo/evodb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index ef8400045a84..df943d7e77a7 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -510,7 +510,7 @@ class CScopedDBTransaction { bool didCommitOrRollback{}; public: - CScopedDBTransaction(Transaction &dbTx) : dbTransaction(dbTx) {} + explicit CScopedDBTransaction(Transaction &dbTx) : dbTransaction(dbTx) {} ~CScopedDBTransaction() { if (!didCommitOrRollback) Rollback(); diff --git a/src/evo/evodb.h b/src/evo/evodb.h index f372ac2a61ec..5c6c8cd61f3f 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -27,7 +27,7 @@ class CEvoDB CurTransaction curDBTransaction; public: - CEvoDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); + explicit CEvoDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); std::unique_ptr BeginTransaction() { From 65b8ddcdd4d3fe801588834708558f8418edac89 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 11:17:29 +0100 Subject: [PATCH 12/20] [DB] Specialize CScopedDBTransaction for evo database >>> backports dash@5b4fe43c2577f3f39fd174c84a8c23c99646903d This has the wanted side effect of proper locking of "cs" inside CommitCurTransaction and RollbackCurTransaction, which was not easily possible to implement in the generic version. This fixes some rare crashes. --- src/dbwrapper.h | 45 --------------------------------------------- src/evo/evodb.cpp | 37 +++++++++++++++++++++++++++++++++++++ src/evo/evodb.h | 28 ++++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 49 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index df943d7e77a7..7988e82bbb0f 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -498,49 +498,4 @@ class CDBTransaction { } }; -template -class CScopedDBTransaction { -public: - typedef CDBTransaction Transaction; - -private: - Transaction &dbTransaction; - std::function commitHandler; - std::function rollbackHandler; - bool didCommitOrRollback{}; - -public: - explicit CScopedDBTransaction(Transaction &dbTx) : dbTransaction(dbTx) {} - ~CScopedDBTransaction() { - if (!didCommitOrRollback) - Rollback(); - } - void Commit() { - assert(!didCommitOrRollback); - didCommitOrRollback = true; - dbTransaction.Commit(); - if (commitHandler) - commitHandler(); - } - void Rollback() { - assert(!didCommitOrRollback); - didCommitOrRollback = true; - dbTransaction.Clear(); - if (rollbackHandler) - rollbackHandler(); - } - - static std::unique_ptr> Begin(Transaction &dbTx) { - assert(dbTx.IsClean()); - return std::make_unique>(dbTx); - } - - void SetCommitHandler(const std::function &h) { - commitHandler = h; - } - void SetRollbackHandler(const std::function &h) { - rollbackHandler = h; - } -}; - #endif // BITCOIN_DBWRAPPER_H diff --git a/src/evo/evodb.cpp b/src/evo/evodb.cpp index 29e3abb9cd9f..01f48ddcec64 100644 --- a/src/evo/evodb.cpp +++ b/src/evo/evodb.cpp @@ -7,6 +7,31 @@ std::unique_ptr evoDb; +CEvoDBScopedCommitter::CEvoDBScopedCommitter(CEvoDB &_evoDB) : + evoDB(_evoDB) +{ +} + +CEvoDBScopedCommitter::~CEvoDBScopedCommitter() +{ + if (!didCommitOrRollback) + Rollback(); +} + +void CEvoDBScopedCommitter::Commit() +{ + assert(!didCommitOrRollback); + didCommitOrRollback = true; + evoDB.CommitCurTransaction(); +} + +void CEvoDBScopedCommitter::Rollback() +{ + assert(!didCommitOrRollback); + didCommitOrRollback = true; + evoDB.RollbackCurTransaction(); +} + CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(fMemory ? "" : (GetDataDir() / "evodb"), nCacheSize, fMemory, fWipe), rootBatch(), @@ -15,6 +40,18 @@ CEvoDB::CEvoDB(size_t nCacheSize, bool fMemory, bool fWipe) : { } +void CEvoDB::CommitCurTransaction() +{ + LOCK(cs); + curDBTransaction.Commit(); +} + +void CEvoDB::RollbackCurTransaction() +{ + LOCK(cs); + curDBTransaction.Clear(); +} + bool CEvoDB::CommitRootTransaction() { assert(curDBTransaction.IsClean()); diff --git a/src/evo/evodb.h b/src/evo/evodb.h index 5c6c8cd61f3f..440225e85d7e 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -12,6 +12,22 @@ static const std::string EVODB_BEST_BLOCK = "b_b"; +class CEvoDB; + +class CEvoDBScopedCommitter +{ +private: + CEvoDB& evoDB; + bool didCommitOrRollback{false}; + +public: + explicit CEvoDBScopedCommitter(CEvoDB& _evoDB); + ~CEvoDBScopedCommitter(); + + void Commit(); + void Rollback(); +}; + class CEvoDB { private: @@ -20,7 +36,6 @@ class CEvoDB typedef CDBTransaction RootTransaction; typedef CDBTransaction CurTransaction; - typedef CScopedDBTransaction ScopedTransaction; CDBBatch rootBatch; RootTransaction rootDBTransaction; @@ -29,11 +44,10 @@ class CEvoDB public: explicit CEvoDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false); - std::unique_ptr BeginTransaction() + std::unique_ptr BeginTransaction() { LOCK(cs); - auto t = ScopedTransaction::Begin(curDBTransaction); - return t; + return std::make_unique(*this); } template @@ -73,6 +87,12 @@ class CEvoDB bool VerifyBestBlock(const uint256& hash); void WriteBestBlock(const uint256& hash); + +private: + // only CEvoDBScopedCommitter is allowed to invoke these + friend class CEvoDBScopedCommitter; + void CommitCurTransaction(); + void RollbackCurTransaction(); }; extern std::unique_ptr evoDb; From cdd44d48764a6a988c694149b4a71b3f697fddc2 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 18:43:46 +0100 Subject: [PATCH 13/20] [Cleanup] Drop unneeded casts --- src/budget/budgetproposal.cpp | 4 ++-- src/budget/budgetproposal.h | 2 +- src/budget/finalizedbudget.h | 2 +- src/masternode-payments.h | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/budget/budgetproposal.cpp b/src/budget/budgetproposal.cpp index f04aac7dae4e..4c35466bb119 100644 --- a/src/budget/budgetproposal.cpp +++ b/src/budget/budgetproposal.cpp @@ -58,7 +58,7 @@ bool CBudgetProposal::ParseBroadcast(CDataStream& broadcast) broadcast >> nBlockStart; broadcast >> nBlockEnd; broadcast >> nAmount; - broadcast >> *(CScriptBase*)(&address); + broadcast >> address; broadcast >> nFeeTXHash; } catch (std::exception& e) { return error("Unable to deserialize proposal broadcast: %s", e.what()); @@ -338,7 +338,7 @@ CDataStream CBudgetProposal::GetBroadcast() const broadcast << nBlockStart; broadcast << nBlockEnd; broadcast << nAmount; - broadcast << *(CScriptBase*)(&address); + broadcast << address; broadcast << nFeeTXHash; return broadcast; } diff --git a/src/budget/budgetproposal.h b/src/budget/budgetproposal.h index d5f64d3bd470..9b889d78ef41 100644 --- a/src/budget/budgetproposal.h +++ b/src/budget/budgetproposal.h @@ -115,7 +115,7 @@ class CBudgetProposal READWRITE(nBlockStart); READWRITE(nBlockEnd); READWRITE(nAmount); - READWRITE(*(CScriptBase*)(&address)); + READWRITE(address); READWRITE(nFeeTXHash); READWRITE(nTime); READWRITE(mapVotes); diff --git a/src/budget/finalizedbudget.h b/src/budget/finalizedbudget.h index 08cc04c1fe2c..cc2725286b54 100644 --- a/src/budget/finalizedbudget.h +++ b/src/budget/finalizedbudget.h @@ -161,7 +161,7 @@ class CTxBudgetPayment template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(*(CScriptBase*)(&payee)); + READWRITE(payee); READWRITE(nAmount); READWRITE(nProposalHash); } diff --git a/src/masternode-payments.h b/src/masternode-payments.h index 7199b9fcbf54..3e042dadb2db 100644 --- a/src/masternode-payments.h +++ b/src/masternode-payments.h @@ -78,7 +78,7 @@ class CMasternodePayee template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(*(CScriptBase*)(&scriptPubKey)); + READWRITE(scriptPubKey); READWRITE(nVotes); } }; @@ -199,7 +199,7 @@ class CMasternodePaymentWinner : public CSignedMessage { READWRITE(vinMasternode); READWRITE(nBlockHeight); - READWRITE(*(CScriptBase*)(&payee)); + READWRITE(payee); READWRITE(vchSig); try { From 7094a6022c382edd8b1ed52f2209753eeb37f988 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 4 Apr 2019 08:19:26 +0200 Subject: [PATCH 14/20] Support passing CDataStream as key into CDBWrapper/CDBBatch/CDBIterator This allow to pre-serialize keys in advance and pass the serialized form into these classes. --- src/dbwrapper.h | 62 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 7988e82bbb0f..91e294874981 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -72,7 +72,14 @@ class CDBBatch CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + Write(ssKey, value); + ssKey.clear(); + } + + template + void Write(const CDataStream& _ssKey, const V& value) + { + leveldb::Slice slKey(_ssKey.data(), _ssKey.size()); CDataStream ssValue(SER_DISK, CLIENT_VERSION); ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); @@ -89,7 +96,6 @@ class CDBBatch // - byte[]: value // The formula below assumes the key and value are both less than 16k. size_estimate += 3 + (slKey.size() > 127) + slKey.size() + (slValue.size() > 127) + slValue.size(); - ssKey.clear(); ssValue.clear(); } @@ -99,7 +105,13 @@ class CDBBatch CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + Erase(ssKey); + ssKey.clear(); + } + + void Erase(const CDataStream& _ssKey) + { + leveldb::Slice slKey(_ssKey.data(), _ssKey.size()); batch.Delete(slKey); @@ -109,7 +121,6 @@ class CDBBatch // - byte[]: key // The formula below assumes the key is less than 16kB. size_estimate += 2 + (slKey.size() > 127) + slKey.size(); - ssKey.clear(); } size_t SizeEstimate() const { return size_estimate; } @@ -135,6 +146,10 @@ class CDBIterator CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; + Seek(ssKey); + } + + void Seek(const CDataStream& ssKey) { leveldb::Slice slKey(ssKey.data(), ssKey.size()); piter->Seek(slKey); } @@ -207,12 +222,17 @@ class CDBWrapper CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false); ~CDBWrapper(); - template - bool Read(const K& key, V& value) const + template + bool ReadDataStream(const K& key, CDataStream& ssValue) const { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; + return ReadDataStream(ssKey, ssValue); + } + + bool ReadDataStream(const CDataStream& ssKey, CDataStream& ssValue) const + { leveldb::Slice slKey(ssKey.data(), ssKey.size()); std::string strValue; @@ -225,6 +245,29 @@ class CDBWrapper } try { CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + } catch (const std::exception&) { + return false; + } + return true; + } + + template + bool Read(const K& key, V& value) const + { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey << key; + return Read(ssKey, value); + } + + template + bool Read(const CDataStream& ssKey, V& value) const + { + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + if (!ReadDataStream(ssKey, ssValue)) { + return false; + } + try { ssValue >> value; } catch (const std::exception&) { return false; @@ -246,7 +289,12 @@ class CDBWrapper CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey(ssKey.data(), ssKey.size()); + return Exists(ssKey); + } + + bool Exists(const CDataStream& key) const + { + leveldb::Slice slKey(key.data(), key.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); From 51920cb2a27d9db24bbfca487ac98fdfcb8b24ab Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 4 Apr 2019 09:58:13 +0200 Subject: [PATCH 15/20] Change CDBTransaction to compare keys by their serialized form ...instead of comparing by using the keys < operator. There are 2 reasons for this: 1. This better mimics the behavior of CDBWrapper. Otherwise there is a chance of discrepancy when it comes to key equality. 2. Next commit will introduce CDBTransactionIterator, which relies on CDBTransaction and CDBWrapper being compatible in the way keys are compared. --- src/dbwrapper.h | 179 +++++++++++++++++------------------------------- 1 file changed, 62 insertions(+), 117 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 91e294874981..b8abab4e5ddd 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -364,162 +364,111 @@ class CDBTransaction { Parent &parent; CommitTarget &commitTarget; - struct KeyHolder { - virtual ~KeyHolder() = default; - virtual bool Less(const KeyHolder &b) const = 0; - virtual void Erase(CommitTarget &commitTarget) = 0; - }; - typedef std::unique_ptr KeyHolderPtr; - - template - struct KeyHolderImpl : KeyHolder { - KeyHolderImpl(const K &_key) - : key(_key) { + struct DataStreamCmp { + static bool less(const CDataStream& a, const CDataStream& b) { + return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); } - virtual bool Less(const KeyHolder &b) const { - auto *b2 = dynamic_cast*>(&b); - return key < b2->key; + bool operator()(const CDataStream& a, const CDataStream& b) const { + return less(a, b); } - virtual void Erase(CommitTarget &commitTarget) { - commitTarget.Erase(key); - } - K key; }; - struct KeyValueHolder { - virtual ~KeyValueHolder() = default; - virtual void Write(CommitTarget &parent) = 0; + struct ValueHolder { + virtual ~ValueHolder() = default; + virtual void Write(const CDataStream& ssKey, CommitTarget &parent) = 0; }; - typedef std::unique_ptr KeyValueHolderPtr; + typedef std::unique_ptr ValueHolderPtr; - template - struct KeyValueHolderImpl : KeyValueHolder { - KeyValueHolderImpl(const KeyHolderImpl &_key, const V &_value) - : key(_key), - value(_value) { } - KeyValueHolderImpl(const KeyHolderImpl &_key, V &&_value) - : key(_key), - value(std::forward(_value)) { } - virtual void Write(CommitTarget &commitTarget) { + template + struct ValueHolderImpl : ValueHolder { + ValueHolderImpl(const V &_value) : value(_value) { } + + virtual void Write(const CDataStream& ssKey, CommitTarget &commitTarget) { // we're moving the value instead of copying it. This means that Write() can only be called once per - // KeyValueHolderImpl instance. Commit() clears the write maps, so this ok. - commitTarget.Write(key.key, std::move(value)); + // ValueHolderImpl instance. Commit() clears the write maps, so this ok. + commitTarget.Write(ssKey, std::move(value)); } - const KeyHolderImpl &key; V value; }; - struct keyCmp { - bool operator()(const KeyHolderPtr &a, const KeyHolderPtr &b) const { - return a->Less(*b); - } - }; - - typedef std::map KeyValueMap; - typedef std::map TypeKeyValueMap; - - TypeKeyValueMap writes; - TypeKeyValueMap deletes; - - template - KeyValueMap *getMapForType(TypeKeyValueMap &m, bool create) { - auto it = m.find(typeid(K)); - if (it != m.end()) { - return &it->second; - } - if (!create) - return nullptr; - auto it2 = m.emplace(typeid(K), KeyValueMap()); - return &it2.first->second; - } - - template - KeyValueMap *getWritesMap(bool create) { - return getMapForType(writes, create); - } - - template - KeyValueMap *getDeletesMap(bool create) { - return getMapForType(deletes, create); + template + static CDataStream KeyToDataStream(const K& key) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey << key; + return ssKey; } - template - void writeImpl(KeyHolderImpl* k, KV&& kv) { - auto k2 = KeyHolderPtr(k); + typedef std::map WritesMap; + typedef std::set DeletesSet; - KeyValueMap *ds = getDeletesMap(false); - if (ds) - ds->erase(k2); - - KeyValueMap *ws = getWritesMap(true); - ws->erase(k2); - ws->emplace(std::make_pair(std::move(k2), std::forward(kv))); - } + WritesMap writes; + DeletesSet deletes; public: CDBTransaction(Parent &_parent, CommitTarget &_commitTarget) : parent(_parent), commitTarget(_commitTarget) {} template void Write(const K& key, const V& v) { - auto k = new KeyHolderImpl(key); - auto kv = std::make_unique>(*k, v); - writeImpl(k, std::move(kv)); + Write(KeyToDataStream(key), v); } - template - void Write(const K& key, V&& v) { - auto k = new KeyHolderImpl(key); - auto kv = std::make_unique::type>>(*k, std::forward(v)); - writeImpl(k, std::move(kv)); + template + void Write(const CDataStream& ssKey, const V& v) { + deletes.erase(ssKey); + auto it = writes.emplace(ssKey, nullptr).first; + it->second = std::make_unique>(v); } template bool Read(const K& key, V& value) { - KeyHolderPtr k(new KeyHolderImpl(key)); + return Read(KeyToDataStream(key), value); + } - KeyValueMap *ds = getDeletesMap(false); - if (ds && ds->count(k)) + template + bool Read(const CDataStream& ssKey, V& value) { + if (deletes.count(ssKey)) { return false; + } - KeyValueMap *ws = getWritesMap(false); - if (ws) { - auto it = ws->find(k); - if (it != ws->end()) { - auto *impl = dynamic_cast *>(it->second.get()); - if (!impl) - return false; - value = impl->value; - return true; + auto it = writes.find(ssKey); + if (it != writes.end()) { + auto *impl = dynamic_cast *>(it->second.get()); + if (!impl) { + throw std::runtime_error("Read called with V != previously written type"); } + value = impl->value; + return true; } - return parent.Read(key, value); + return parent.Read(ssKey, value); } template bool Exists(const K& key) { - KeyHolderPtr k(new KeyHolderImpl(key)); + return Exists(KeyToDataStream(key)); + } - KeyValueMap *ds = getDeletesMap(false); - if (ds && ds->count(k)) + bool Exists(const CDataStream& ssKey) { + if (deletes.count(ssKey)) { return false; + } - KeyValueMap *ws = getWritesMap(false); - if (ws && ws->count(k)) + if (writes.count(ssKey)) { return true; + } - return parent.Exists(key); + return parent.Exists(ssKey); } template void Erase(const K& key) { - KeyHolderPtr k(new KeyHolderImpl(key)); + return Erase(KeyToDataStream(key)); + } - KeyValueMap *ws = getWritesMap(false); - if (ws) - ws->erase(k); - KeyValueMap *ds = getDeletesMap(true); - ds->emplace(std::move(k), nullptr); + void Erase(const CDataStream& ssKey) { + writes.erase(ssKey); + deletes.emplace(ssKey); } void Clear() { @@ -528,15 +477,11 @@ class CDBTransaction { } void Commit() { - for (auto &p : deletes) { - for (auto &p2 : p.second) { - p2.first->Erase(commitTarget); - } + for (const auto &k : deletes) { + commitTarget.Erase(k); } for (auto &p : writes) { - for (auto &p2 : p.second) { - p2.second->Write(commitTarget); - } + p.second->Write(p.first, commitTarget); } Clear(); } From f259774db579dd03b22307d3b069a22c10975c70 Mon Sep 17 00:00:00 2001 From: Alexander Block Date: Thu, 4 Apr 2019 10:03:56 +0200 Subject: [PATCH 16/20] Implement CDBTransactionIterator This iterator allows merged iteration of the key/values from the parent and the not-yet-committed key/values from the transaction. This also works for nested transactions (as used in CEvoDB). It's interface mimics CDBIterator. --- src/dbwrapper.h | 195 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 185 insertions(+), 10 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b8abab4e5ddd..b643dfe13e6e 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -69,7 +69,6 @@ class CDBBatch template void Write(const K& key, const V& value) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; Write(ssKey, value); @@ -102,7 +101,6 @@ class CDBBatch template void Erase(const K& key) { - CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; Erase(ssKey); @@ -157,9 +155,8 @@ class CDBIterator void Next(); template bool GetKey(K& key) { - leveldb::Slice slKey = piter->key(); try { - CDataStream ssKey(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + CDataStream ssKey = GetKey(); ssKey >> key; } catch(const std::exception& e) { return false; @@ -167,6 +164,11 @@ class CDBIterator return true; } + CDataStream GetKey() { + leveldb::Slice slKey = piter->key(); + return CDataStream(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); + } + unsigned int GetKeySize() { return piter->key().size(); } @@ -243,11 +245,8 @@ class CDBWrapper LogPrintf("LevelDB read failure: %s\n", status.ToString()); dbwrapper_private::HandleError(status); } - try { - CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); - } catch (const std::exception&) { - return false; - } + CDataStream ssValueTmp(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION); + ssValue = std::move(ssValueTmp); return true; } @@ -356,17 +355,186 @@ class CDBWrapper return size; } + /** + * Compact a certain range of keys in the database. + */ + template + void CompactRange(const K& key_begin, const K& key_end) const + { + CDataStream ssKey1(SER_DISK, CLIENT_VERSION), ssKey2(SER_DISK, CLIENT_VERSION); + ssKey1.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); + ssKey1 << key_begin; + ssKey2 << key_end; + leveldb::Slice slKey1(ssKey1.data(), ssKey1.size()); + leveldb::Slice slKey2(ssKey2.data(), ssKey2.size()); + pdb->CompactRange(&slKey1, &slKey2); + } + + void CompactFull() const + { + pdb->CompactRange(nullptr, nullptr); + } + +}; + +template +class CDBTransactionIterator +{ +private: + CDBTransaction& transaction; + + typedef typename std::remove_pointer::type ParentIterator; + + // We maintain 2 iterators, one for the transaction and one for the parent + // At all times, only one of both provides the current value. The decision is made by comparing the current keys + // of both iterators, so that always the smaller key is the current one. On Next(), the previously chosen iterator + // is advanced. + typename CDBTransaction::WritesMap::iterator transactionIt; + std::unique_ptr parentIt; + CDataStream parentKey; + bool curIsParent{false}; + +public: + CDBTransactionIterator(CDBTransaction& _transaction) : + transaction(_transaction), + parentKey(SER_DISK, CLIENT_VERSION) + { + transactionIt = transaction.writes.end(); + parentIt = std::unique_ptr(transaction.parent.NewIterator()); + } + + void SeekToFirst() { + transactionIt = transaction.writes.begin(); + parentIt->SeekToFirst(); + SkipDeletedAndOverwritten(); + DecideCur(); + } + + template + void Seek(const K& key) { + Seek(CDBTransaction::KeyToDataStream(key)); + } + + void Seek(const CDataStream& ssKey) { + transactionIt = transaction.writes.lower_bound(ssKey); + parentIt->Seek(ssKey); + SkipDeletedAndOverwritten(); + DecideCur(); + } + + bool Valid() { + return transactionIt != transaction.writes.end() || parentIt->Valid(); + } + + void Next() { + if (transactionIt == transaction.writes.end() && !parentIt->Valid()) { + return; + } + if (curIsParent) { + assert(parentIt->Valid()); + parentIt->Next(); + SkipDeletedAndOverwritten(); + } else { + assert(transactionIt != transaction.writes.end()); + ++transactionIt; + } + DecideCur(); + } + + template + bool GetKey(K& key) { + if (!Valid()) { + return false; + } + + if (curIsParent) { + return parentIt->GetKey(key); + } else { + try { + // TODO try to avoid this copy (we need a stream that allows reading from external buffers) + CDataStream ssKey = transactionIt->first; + ssKey >> key; + } catch (const std::exception&) { + return false; + } + return true; + } + } + + CDataStream GetKey() { + if (!Valid()) { + return CDataStream(SER_DISK, CLIENT_VERSION); + } + if (curIsParent) { + return parentIt->GetKey(); + } else { + return transactionIt->first; + } + } + + unsigned int GetKeySize() { + if (!Valid()) { + return 0; + } + if (curIsParent) { + return parentIt->GetKeySize(); + } else { + return transactionIt->first.vKey.size(); + } + } + + template + bool GetValue(V& value) { + if (!Valid()) { + return false; + } + if (curIsParent) { + return transaction.Read(parentKey, value); + } else { + return transaction.Read(transactionIt->first, value); + } + }; + +private: + void SkipDeletedAndOverwritten() { + while (parentIt->Valid()) { + parentKey = parentIt->GetKey(); + if (!transaction.deletes.count(parentKey) && !transaction.writes.count(parentKey)) { + break; + } + parentIt->Next(); + } + } + + void DecideCur() { + if (transactionIt != transaction.writes.end() && !parentIt->Valid()) { + curIsParent = false; + } else if (transactionIt == transaction.writes.end() && parentIt->Valid()) { + curIsParent = true; + } else if (transactionIt != transaction.writes.end() && parentIt->Valid()) { + if (CDBTransaction::DataStreamCmp::less(transactionIt->first, parentKey)) { + curIsParent = false; + } else { + curIsParent = true; + } + } + } }; template class CDBTransaction { + friend class CDBTransactionIterator; + protected: Parent &parent; CommitTarget &commitTarget; struct DataStreamCmp { static bool less(const CDataStream& a, const CDataStream& b) { - return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); + return std::lexicographical_compare( + (const uint8_t*)a.data(), (const uint8_t*)a.data() + a.size(), + (const uint8_t*)b.data(), (const uint8_t*)b.data() + b.size()); } bool operator()(const CDataStream& a, const CDataStream& b) const { return less(a, b); @@ -489,6 +657,13 @@ class CDBTransaction { bool IsClean() { return writes.empty() && deletes.empty(); } + + CDBTransactionIterator* NewIterator() { + return new CDBTransactionIterator(*this); + } + std::unique_ptr> NewIteratorUniquePtr() { + return std::make_unique>(*this); + } }; #endif // BITCOIN_DBWRAPPER_H From ed0d4f99d96d48d6025c81ff5a3df2e621a02865 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 21:01:13 +0100 Subject: [PATCH 17/20] Track memory usage in CDBTransaction and CEvoDB >>> backports dash@1b3d0d9a347f1e40e9efa246ff3691c8e995383f --- src/dbwrapper.h | 41 ++++++++++++++++++++++++++++++++++++----- src/evo/evodb.h | 5 +++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b643dfe13e6e..b6480c9fbf44 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -529,6 +529,7 @@ class CDBTransaction { protected: Parent &parent; CommitTarget &commitTarget; + ssize_t memoryUsage{0}; // signed, just in case we made an error in the calculations so that we don't get an overflow struct DataStreamCmp { static bool less(const CDataStream& a, const CDataStream& b) { @@ -542,6 +543,8 @@ class CDBTransaction { }; struct ValueHolder { + size_t memoryUsage; + ValueHolder(size_t _memoryUsage) : memoryUsage(_memoryUsage) {} virtual ~ValueHolder() = default; virtual void Write(const CDataStream& ssKey, CommitTarget &parent) = 0; }; @@ -549,7 +552,7 @@ class CDBTransaction { template struct ValueHolderImpl : ValueHolder { - ValueHolderImpl(const V &_value) : value(_value) { } + ValueHolderImpl(const V &_value, size_t _memoryUsage) : ValueHolder(_memoryUsage), value(_value) {} virtual void Write(const CDataStream& ssKey, CommitTarget &commitTarget) { // we're moving the value instead of copying it. This means that Write() can only be called once per @@ -583,9 +586,17 @@ class CDBTransaction { template void Write(const CDataStream& ssKey, const V& v) { - deletes.erase(ssKey); + auto valueMemoryUsage = ::GetSerializeSize(v, SER_DISK, CLIENT_VERSION); + if (deletes.erase(ssKey)) { + memoryUsage -= ssKey.size(); + } auto it = writes.emplace(ssKey, nullptr).first; - it->second = std::make_unique>(v); + if (it->second) { + memoryUsage -= ssKey.size() + it->second->memoryUsage; + } + it->second = std::make_unique>(v, valueMemoryUsage); + + memoryUsage += ssKey.size() + valueMemoryUsage; } template @@ -635,13 +646,20 @@ class CDBTransaction { } void Erase(const CDataStream& ssKey) { - writes.erase(ssKey); - deletes.emplace(ssKey); + auto it = writes.find(ssKey); + if (it != writes.end()) { + memoryUsage -= ssKey.size() + it->second->memoryUsage; + writes.erase(it); + } + if (deletes.emplace(ssKey).second) { + memoryUsage += ssKey.size(); + } } void Clear() { writes.clear(); deletes.clear(); + memoryUsage = 0; } void Commit() { @@ -658,6 +676,19 @@ class CDBTransaction { return writes.empty() && deletes.empty(); } + size_t GetMemoryUsage() const { + if (memoryUsage < 0) { + // something went wrong when we accounted/calculated used memory... + static volatile bool didPrint = false; + if (!didPrint) { + LogPrintf("CDBTransaction::%s -- negative memoryUsage (%d)", __func__, memoryUsage); + didPrint = true; + } + return 0; + } + return (size_t)memoryUsage; + } + CDBTransactionIterator* NewIterator() { return new CDBTransactionIterator(*this); } diff --git a/src/evo/evodb.h b/src/evo/evodb.h index 440225e85d7e..700537e54588 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -83,6 +83,11 @@ class CEvoDB return db; } + size_t GetMemoryUsage() + { + return rootDBTransaction.GetMemoryUsage(); + } + bool CommitRootTransaction(); bool VerifyBestBlock(const uint256& hash); From 304fc466813f95eb24509ec5232b64486b9f7d70 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 21:17:24 +0100 Subject: [PATCH 18/20] Take memory used by CEvoDB/CDBTransaction into account when flushing >>> backports dash@849bf3c76e62a980e89f2be2877ea2cc3728017c --- src/validation.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/validation.cpp b/src/validation.cpp index b9a156950c35..e518b9adcda4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1827,6 +1827,7 @@ bool static FlushStateToDisk(CValidationState& state, FlushStateMode mode) } int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + cacheSize += evoDb->GetMemoryUsage(); int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now // (not in the middle of a block processing). @@ -1930,6 +1931,7 @@ void static UpdateTip(CBlockIndex* pindexNew) pChainTip->GetBlockHash().GetHex(), pChainTip->nHeight, pChainTip->nVersion, log(pChainTip->nChainWork.getdouble()) / log(2.0), (unsigned long)pChainTip->nChainTx, DateTimeStrFormat("%Y-%m-%d %H:%M:%S", pChainTip->GetBlockTime()), Checkpoints::GuessVerificationProgress(pChainTip), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + LogPrintf("%s: evodb_cache=%.1fMiB", __func__, evoDb->GetMemoryUsage() * (1.0 / (1<<20))); // Check the version of the last 100 blocks to see if we need to upgrade: static bool fWarned = false; From 8850b3e8c1fdf4aa67ddf29a3bbd129c012296fa Mon Sep 17 00:00:00 2001 From: random-zebra Date: Thu, 18 Mar 2021 21:35:39 +0100 Subject: [PATCH 19/20] [Refactoring] Make CEvoDB mutex public it will be locked from outside when iterating mined commitments (ref. dash@c68b5f68aa41881c3bf36d8881b60b6855868605) --- src/evo/evodb.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/evo/evodb.h b/src/evo/evodb.h index 700537e54588..b5bf1ce6a6a3 100644 --- a/src/evo/evodb.h +++ b/src/evo/evodb.h @@ -30,8 +30,10 @@ class CEvoDBScopedCommitter class CEvoDB { -private: +public: RecursiveMutex cs; + +private: CDBWrapper db; typedef CDBTransaction RootTransaction; @@ -50,6 +52,12 @@ class CEvoDB return std::make_unique(*this); } + CurTransaction& GetCurTransaction() + { + AssertLockHeld(cs); // lock must be held from outside as long as the DB transaction is used + return curDBTransaction; + } + template bool Read(const K& key, V& value) { From c27bbed57b638881d68778940e8917c1b965ee8b Mon Sep 17 00:00:00 2001 From: random-zebra Date: Fri, 19 Mar 2021 00:05:19 +0100 Subject: [PATCH 20/20] [Cleanup] functions impl: branch on new line --- src/dbwrapper.h | 109 +++++++++++++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 38 deletions(-) diff --git a/src/dbwrapper.h b/src/dbwrapper.h index b6480c9fbf44..3889651a9d71 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -140,21 +140,24 @@ class CDBIterator void SeekToFirst(); - template void Seek(const K& key) { + template void Seek(const K& key) + { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; Seek(ssKey); } - void Seek(const CDataStream& ssKey) { + void Seek(const CDataStream& ssKey) + { leveldb::Slice slKey(ssKey.data(), ssKey.size()); piter->Seek(slKey); } void Next(); - template bool GetKey(K& key) { + template bool GetKey(K& key) + { try { CDataStream ssKey = GetKey(); ssKey >> key; @@ -164,16 +167,19 @@ class CDBIterator return true; } - CDataStream GetKey() { + CDataStream GetKey() + { leveldb::Slice slKey = piter->key(); return CDataStream(slKey.data(), slKey.data() + slKey.size(), SER_DISK, CLIENT_VERSION); } - unsigned int GetKeySize() { + unsigned int GetKeySize() + { return piter->key().size(); } - template bool GetValue(V& value) { + template bool GetValue(V& value) + { leveldb::Slice slValue = piter->value(); try { CDataStream ssValue(slValue.data(), slValue.data() + slValue.size(), SER_DISK, CLIENT_VERSION); @@ -184,7 +190,8 @@ class CDBIterator return true; } - unsigned int GetValueSize() { + unsigned int GetValueSize() + { return piter->value().size(); } @@ -404,7 +411,8 @@ class CDBTransactionIterator parentIt = std::unique_ptr(transaction.parent.NewIterator()); } - void SeekToFirst() { + void SeekToFirst() + { transactionIt = transaction.writes.begin(); parentIt->SeekToFirst(); SkipDeletedAndOverwritten(); @@ -412,22 +420,26 @@ class CDBTransactionIterator } template - void Seek(const K& key) { + void Seek(const K& key) + { Seek(CDBTransaction::KeyToDataStream(key)); } - void Seek(const CDataStream& ssKey) { + void Seek(const CDataStream& ssKey) + { transactionIt = transaction.writes.lower_bound(ssKey); parentIt->Seek(ssKey); SkipDeletedAndOverwritten(); DecideCur(); } - bool Valid() { + bool Valid() + { return transactionIt != transaction.writes.end() || parentIt->Valid(); } - void Next() { + void Next() + { if (transactionIt == transaction.writes.end() && !parentIt->Valid()) { return; } @@ -443,7 +455,8 @@ class CDBTransactionIterator } template - bool GetKey(K& key) { + bool GetKey(K& key) + { if (!Valid()) { return false; } @@ -462,7 +475,8 @@ class CDBTransactionIterator } } - CDataStream GetKey() { + CDataStream GetKey() + { if (!Valid()) { return CDataStream(SER_DISK, CLIENT_VERSION); } @@ -473,7 +487,8 @@ class CDBTransactionIterator } } - unsigned int GetKeySize() { + unsigned int GetKeySize() + { if (!Valid()) { return 0; } @@ -485,7 +500,8 @@ class CDBTransactionIterator } template - bool GetValue(V& value) { + bool GetValue(V& value) + { if (!Valid()) { return false; } @@ -497,7 +513,8 @@ class CDBTransactionIterator }; private: - void SkipDeletedAndOverwritten() { + void SkipDeletedAndOverwritten() + { while (parentIt->Valid()) { parentKey = parentIt->GetKey(); if (!transaction.deletes.count(parentKey) && !transaction.writes.count(parentKey)) { @@ -507,7 +524,8 @@ class CDBTransactionIterator } } - void DecideCur() { + void DecideCur() + { if (transactionIt != transaction.writes.end() && !parentIt->Valid()) { curIsParent = false; } else if (transactionIt == transaction.writes.end() && parentIt->Valid()) { @@ -532,14 +550,13 @@ class CDBTransaction { ssize_t memoryUsage{0}; // signed, just in case we made an error in the calculations so that we don't get an overflow struct DataStreamCmp { - static bool less(const CDataStream& a, const CDataStream& b) { + static bool less(const CDataStream& a, const CDataStream& b) + { return std::lexicographical_compare( (const uint8_t*)a.data(), (const uint8_t*)a.data() + a.size(), (const uint8_t*)b.data(), (const uint8_t*)b.data() + b.size()); } - bool operator()(const CDataStream& a, const CDataStream& b) const { - return less(a, b); - } + bool operator()(const CDataStream& a, const CDataStream& b) const { return less(a, b); } }; struct ValueHolder { @@ -554,7 +571,8 @@ class CDBTransaction { struct ValueHolderImpl : ValueHolder { ValueHolderImpl(const V &_value, size_t _memoryUsage) : ValueHolder(_memoryUsage), value(_value) {} - virtual void Write(const CDataStream& ssKey, CommitTarget &commitTarget) { + virtual void Write(const CDataStream& ssKey, CommitTarget &commitTarget) + { // we're moving the value instead of copying it. This means that Write() can only be called once per // ValueHolderImpl instance. Commit() clears the write maps, so this ok. commitTarget.Write(ssKey, std::move(value)); @@ -563,7 +581,8 @@ class CDBTransaction { }; template - static CDataStream KeyToDataStream(const K& key) { + static CDataStream KeyToDataStream(const K& key) + { CDataStream ssKey(SER_DISK, CLIENT_VERSION); ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; @@ -580,12 +599,14 @@ class CDBTransaction { CDBTransaction(Parent &_parent, CommitTarget &_commitTarget) : parent(_parent), commitTarget(_commitTarget) {} template - void Write(const K& key, const V& v) { + void Write(const K& key, const V& v) + { Write(KeyToDataStream(key), v); } template - void Write(const CDataStream& ssKey, const V& v) { + void Write(const CDataStream& ssKey, const V& v) + { auto valueMemoryUsage = ::GetSerializeSize(v, SER_DISK, CLIENT_VERSION); if (deletes.erase(ssKey)) { memoryUsage -= ssKey.size(); @@ -600,12 +621,14 @@ class CDBTransaction { } template - bool Read(const K& key, V& value) { + bool Read(const K& key, V& value) + { return Read(KeyToDataStream(key), value); } template - bool Read(const CDataStream& ssKey, V& value) { + bool Read(const CDataStream& ssKey, V& value) + { if (deletes.count(ssKey)) { return false; } @@ -624,11 +647,13 @@ class CDBTransaction { } template - bool Exists(const K& key) { + bool Exists(const K& key) + { return Exists(KeyToDataStream(key)); } - bool Exists(const CDataStream& ssKey) { + bool Exists(const CDataStream& ssKey) + { if (deletes.count(ssKey)) { return false; } @@ -641,11 +666,13 @@ class CDBTransaction { } template - void Erase(const K& key) { + void Erase(const K& key) + { return Erase(KeyToDataStream(key)); } - void Erase(const CDataStream& ssKey) { + void Erase(const CDataStream& ssKey) + { auto it = writes.find(ssKey); if (it != writes.end()) { memoryUsage -= ssKey.size() + it->second->memoryUsage; @@ -656,13 +683,15 @@ class CDBTransaction { } } - void Clear() { + void Clear() + { writes.clear(); deletes.clear(); memoryUsage = 0; } - void Commit() { + void Commit() + { for (const auto &k : deletes) { commitTarget.Erase(k); } @@ -672,11 +701,13 @@ class CDBTransaction { Clear(); } - bool IsClean() { + bool IsClean() + { return writes.empty() && deletes.empty(); } - size_t GetMemoryUsage() const { + size_t GetMemoryUsage() const + { if (memoryUsage < 0) { // something went wrong when we accounted/calculated used memory... static volatile bool didPrint = false; @@ -689,10 +720,12 @@ class CDBTransaction { return (size_t)memoryUsage; } - CDBTransactionIterator* NewIterator() { + CDBTransactionIterator* NewIterator() + { return new CDBTransactionIterator(*this); } - std::unique_ptr> NewIteratorUniquePtr() { + std::unique_ptr> NewIteratorUniquePtr() + { return std::make_unique>(*this); } };