test: add comprehensive unit tests for SML caching mechanism

src/Makefile.am

@@ -195,6 +195,7 @@ BITCOIN_CORE_H = \
   evo/netinfo.h \
   evo/providertx.h \
   evo/simplifiedmns.h \
+  evo/smldiff.h \
   evo/specialtx.h \
   evo/specialtxman.h \
   dsnotificationinterface.h \
@@ -467,6 +468,7 @@ libbitcoin_node_a_SOURCES = \
   evo/mnhftx.cpp \
   evo/providertx.cpp \
   evo/simplifiedmns.cpp \
+  evo/smldiff.cpp \
   evo/specialtx.cpp \
   evo/specialtxman.cpp \
   flatfile.cpp \

src/evo/cbtx.h

@@ -14,7 +14,9 @@
 class BlockValidationState;
 class CBlock;
 class CBlockIndex;
+class CDeterministicMNList;
 class TxValidationState;
+
 namespace llmq {
 class CQuorumBlockProcessor;
 }// namespace llmq

src/evo/core_write.cpp

@@ -8,6 +8,8 @@
 #include <evo/mnhftx.h>
 #include <evo/netinfo.h>
 #include <evo/providertx.h>
+#include <evo/simplifiedmns.h>
+#include <evo/smldiff.h>
 #include <llmq/commitment.h>
 
 #include <univalue.h>
@@ -143,3 +145,94 @@
     ret.pushKV("commitment", commitment.ToJson());
     return ret;
 }
+
+[[nodiscard]] UniValue CSimplifiedMNListEntry::ToJson(bool extended) const
+{
+    UniValue obj(UniValue::VOBJ);
+    obj.pushKV("nVersion", nVersion);
+    obj.pushKV("nType", ToUnderlying(nType));
+    obj.pushKV("proRegTxHash", proRegTxHash.ToString());
+    obj.pushKV("confirmedHash", confirmedHash.ToString());
+    obj.pushKV("service", netInfo->GetPrimary().ToStringAddrPort());
+    obj.pushKV("pubKeyOperator", pubKeyOperator.ToString());
+    obj.pushKV("votingAddress", EncodeDestination(PKHash(keyIDVoting)));
+    obj.pushKV("isValid", isValid);
+    if (nType == MnType::Evo) {
+        obj.pushKV("platformHTTPPort", platformHTTPPort);
+        obj.pushKV("platformNodeID", platformNodeID.ToString());
+    }
+
+    if (extended) {
+        CTxDestination dest;
+        if (ExtractDestination(scriptPayout, dest)) {
+            obj.pushKV("payoutAddress", EncodeDestination(dest));
+        }
+        if (ExtractDestination(scriptOperatorPayout, dest)) {
+            obj.pushKV("operatorPayoutAddress", EncodeDestination(dest));
+        }
+    }
+    return obj;
+}
+
+[[nodiscard]] UniValue CSimplifiedMNListDiff::ToJson(bool extended) const
+{
+    UniValue obj(UniValue::VOBJ);
+
+    obj.pushKV("nVersion", nVersion);
+    obj.pushKV("baseBlockHash", baseBlockHash.ToString());
+    obj.pushKV("blockHash", blockHash.ToString());
+
+    CDataStream ssCbTxMerkleTree(SER_NETWORK, PROTOCOL_VERSION);
+    ssCbTxMerkleTree << cbTxMerkleTree;
+    obj.pushKV("cbTxMerkleTree", HexStr(ssCbTxMerkleTree));
+
+    obj.pushKV("cbTx", EncodeHexTx(*cbTx));
+
+    UniValue deletedMNsArr(UniValue::VARR);
+    for (const auto& h : deletedMNs) {
+        deletedMNsArr.push_back(h.ToString());
+    }
+    obj.pushKV("deletedMNs", deletedMNsArr);
+
+    UniValue mnListArr(UniValue::VARR);
+    for (const auto& e : mnList) {
+        mnListArr.push_back(e.ToJson(extended));
+    }
+    obj.pushKV("mnList", mnListArr);
+
+    UniValue deletedQuorumsArr(UniValue::VARR);
+    for (const auto& e : deletedQuorums) {
+        UniValue eObj(UniValue::VOBJ);
+        eObj.pushKV("llmqType", e.first);
+        eObj.pushKV("quorumHash", e.second.ToString());
+        deletedQuorumsArr.push_back(eObj);
+    }
+    obj.pushKV("deletedQuorums", deletedQuorumsArr);
+
+    UniValue newQuorumsArr(UniValue::VARR);
+    for (const auto& e : newQuorums) {
+        newQuorumsArr.push_back(e.ToJson());
+    }
+    obj.pushKV("newQuorums", newQuorumsArr);
+
+    // Do not assert special tx type here since this can be called prior to DIP0003 activation
+    if (const auto opt_cbTxPayload = GetTxPayload<CCbTx>(*cbTx, /*assert_type=*/false)) {
+        obj.pushKV("merkleRootMNList", opt_cbTxPayload->merkleRootMNList.ToString());
+        if (opt_cbTxPayload->nVersion >= CCbTx::Version::MERKLE_ROOT_QUORUMS) {
+            obj.pushKV("merkleRootQuorums", opt_cbTxPayload->merkleRootQuorums.ToString());
+        }
+    }
+
+    UniValue quorumsCLSigsArr(UniValue::VARR);
+    for (const auto& [signature, quorumsIndexes] : quorumsCLSigs) {
+        UniValue j(UniValue::VOBJ);
+        UniValue idxArr(UniValue::VARR);
+        for (const auto& idx : quorumsIndexes) {
+            idxArr.push_back(idx);
+        }
+        j.pushKV(signature.ToString(), idxArr);
+        quorumsCLSigsArr.push_back(j);
+    }
+    obj.pushKV("quorumsCLSigs", quorumsCLSigsArr);
+    return obj;
+}

src/evo/deterministicmns.cpp

@@ -7,6 +7,7 @@
 #include <evo/dmnstate.h>
 #include <evo/evodb.h>
 #include <evo/providertx.h>
+#include <evo/simplifiedmns.h>
 #include <evo/specialtx.h>
 #include <index/txindex.h>
 
@@ -36,6 +37,14 @@ uint64_t CDeterministicMN::GetInternalId() const
     return internalId;
 }
 
+CSimplifiedMNListEntry CDeterministicMN::to_sml_entry() const
+{
+    const CDeterministicMNState& state{*pdmnState};
+    return CSimplifiedMNListEntry(proTxHash, state.confirmedHash, state.netInfo, state.pubKeyOperator,
+                                  state.keyIDVoting, !state.IsBanned(), state.platformHTTPPort, state.platformNodeID,
+                                  state.scriptPayout, state.scriptOperatorPayout, state.nVersion, nType);
+}
+
 std::string CDeterministicMN::ToString() const
 {
     return strprintf("CDeterministicMN(proTxHash=%s, collateralOutpoint=%s, nOperatorReward=%f, state=%s", proTxHash.ToString(), collateralOutpoint.ToStringShort(), (double)nOperatorReward / 100, pdmnState->ToString());
@@ -258,6 +267,22 @@ std::vector<CDeterministicMNCPtr> CDeterministicMNList::GetProjectedMNPayees(gsl
     return result;
 }
 
+gsl::not_null<std::shared_ptr<const CSimplifiedMNList>> CDeterministicMNList::to_sml() const
+{
+    LOCK(m_cached_sml_mutex);
+    if (!m_cached_sml) {
+        std::vector<std::unique_ptr<CSimplifiedMNListEntry>> sml_entries;
+        sml_entries.reserve(mnMap.size());
+
+        ForEachMN(false, [&sml_entries](auto& dmn) {
+            sml_entries.emplace_back(std::make_unique<CSimplifiedMNListEntry>(dmn.to_sml_entry()));
+        });
+        m_cached_sml = std::make_shared<CSimplifiedMNList>(std::move(sml_entries));
+    }
+
+    return m_cached_sml;
+}
+
 int CDeterministicMNList::CalcMaxPoSePenalty() const
 {
     // Maximum PoSe penalty is dynamic and equals the number of registered MNs
@@ -443,6 +468,7 @@ void CDeterministicMNList::AddMN(const CDeterministicMNCPtr& dmn, bool fBumpTota
 
     mnMap = mnMap.set(dmn->proTxHash, dmn);
     mnInternalIdMap = mnInternalIdMap.set(dmn->GetInternalId(), dmn->proTxHash);
+    InvalidateSMLCache();
     if (fBumpTotalCount) {
         // nTotalRegisteredCount acts more like a checkpoint, not as a limit,
         nTotalRegisteredCount = std::max(dmn->GetInternalId() + 1, (uint64_t)nTotalRegisteredCount);
@@ -514,6 +540,10 @@ void CDeterministicMNList::UpdateMN(const CDeterministicMN& oldDmn, const std::s
 
     dmn->pdmnState = pdmnState;
     mnMap = mnMap.set(oldDmn.proTxHash, dmn);
+    LOCK(m_cached_sml_mutex);
+    if (m_cached_sml && oldDmn.to_sml_entry() != dmn->to_sml_entry()) {
+        m_cached_sml = nullptr;
+    }
 }
 
 void CDeterministicMNList::UpdateMN(const uint256& proTxHash, const std::shared_ptr<const CDeterministicMNState>& pdmnState)
@@ -585,6 +615,7 @@ void CDeterministicMNList::RemoveMN(const uint256& proTxHash)
 
     mnMap = mnMap.erase(proTxHash);
     mnInternalIdMap = mnInternalIdMap.erase(dmn->GetInternalId());
+    InvalidateSMLCache();
 }
 
 bool CDeterministicMNManager::ProcessBlock(const CBlock& block, gsl::not_null<const CBlockIndex*> pindex,
@@ -604,6 +635,8 @@ bool CDeterministicMNManager::ProcessBlock(const CBlock& block, gsl::not_null<co
     int nHeight = pindex->nHeight;
 
     try {
+        newList.to_sml(); // to populate the SML cache
+
         LOCK(cs);
 
         oldList = GetListForBlockInternal(pindex->pprev);
@@ -619,6 +652,7 @@ bool CDeterministicMNManager::ProcessBlock(const CBlock& block, gsl::not_null<co
 
         diff.nHeight = pindex->nHeight;
         mnListDiffsCache.emplace(pindex->GetBlockHash(), diff);
+        mnListsCache.emplace(newList.GetBlockHash(), newList);
     } catch (const std::exception& e) {
         LogPrintf("CDeterministicMNManager::%s -- internal error: %s\n", __func__, e.what());
         return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "failed-dmn-block");
@@ -752,7 +786,10 @@ CDeterministicMNList CDeterministicMNManager::GetListForBlockInternal(gsl::not_n
     if (tipIndex) {
         // always keep a snapshot for the tip
         if (snapshot.GetBlockHash() == tipIndex->GetBlockHash()) {
-            mnListsCache.emplace(snapshot.GetBlockHash(), snapshot);
+            auto hash = snapshot.GetBlockHash();
+            if (mnListsCache.find(hash) == mnListsCache.end()) {
+                mnListsCache.emplace(snapshot.GetBlockHash(), snapshot);
+            }
         } else {
             // keep snapshots for yet alive quorums
             if (ranges::any_of(Params().GetConsensus().llmqs,

src/evo/deterministicmns.h

@@ -30,6 +30,8 @@ class CBlock;
 class CBlockIndex;
 class CCoinsViewCache;
 class CEvoDB;
+class CSimplifiedMNList;
+class CSimplifiedMNListEntry;
 class TxValidationState;
 
 extern RecursiveMutex cs_main;
@@ -79,6 +81,7 @@ class CDeterministicMN
 
     [[nodiscard]] uint64_t GetInternalId() const;
 
+    [[nodiscard]] CSimplifiedMNListEntry to_sml_entry() const;
     [[nodiscard]] std::string ToString() const;
     [[nodiscard]] UniValue ToJson() const;
 };
@@ -145,6 +148,22 @@ class CDeterministicMNList
     // we keep track of this as checking for duplicates would otherwise be painfully slow
     MnUniquePropertyMap mnUniquePropertyMap;
 
+    // This SML could be null
+    // This cache is used to improve performance and meant to be reused
+    // for multiple CDeterministicMNList until mnMap is actually changed.
+    // Calls of AddMN, RemoveMN and (in some cases) UpdateMN reset this cache;
+    // it happens also for indirect calls such as ApplyDiff
+    // Thread safety: Protected by its own mutex for thread-safe access
+    mutable Mutex m_cached_sml_mutex;
+    mutable std::shared_ptr<const CSimplifiedMNList> m_cached_sml GUARDED_BY(m_cached_sml_mutex);
+
+    // Private helper method to invalidate SML cache
+    void InvalidateSMLCache()
+    {
+        LOCK(m_cached_sml_mutex);
+        m_cached_sml = nullptr;
+    }
+
 public:
     CDeterministicMNList() = default;
     explicit CDeterministicMNList(const uint256& _blockHash, int _height, uint32_t _totalRegisteredCount) :
@@ -155,6 +174,36 @@ class CDeterministicMNList
         assert(nHeight >= 0);
     }
 
+    // Copy constructor
+    CDeterministicMNList(const CDeterministicMNList& other) :
+        blockHash(other.blockHash),
+        nHeight(other.nHeight),
+        nTotalRegisteredCount(other.nTotalRegisteredCount),
+        mnMap(other.mnMap),
+        mnInternalIdMap(other.mnInternalIdMap),
+        mnUniquePropertyMap(other.mnUniquePropertyMap)
+    {
+        LOCK(other.m_cached_sml_mutex);
+        m_cached_sml = other.m_cached_sml;
+    }
+
+    // Assignment operator
+    CDeterministicMNList& operator=(const CDeterministicMNList& other)
+    {
+        if (this != &other) {
+            blockHash = other.blockHash;
+            nHeight = other.nHeight;
+            nTotalRegisteredCount = other.nTotalRegisteredCount;
+            mnMap = other.mnMap;
+            mnInternalIdMap = other.mnInternalIdMap;
+            mnUniquePropertyMap = other.mnUniquePropertyMap;
+
+            LOCK2(m_cached_sml_mutex, other.m_cached_sml_mutex);
+            m_cached_sml = other.m_cached_sml;
+        }
+        return *this;
+    }
+
     template <typename Stream, typename Operation>
     inline void SerializationOpBase(Stream& s, Operation ser_action)
     {
@@ -195,6 +244,7 @@ class CDeterministicMNList
         mnMap = MnMap();
         mnUniquePropertyMap = MnUniquePropertyMap();
         mnInternalIdMap = MnInternalIdMap();
+        InvalidateSMLCache();
     }
 
     [[nodiscard]] size_t GetAllMNsCount() const
@@ -314,6 +364,12 @@ class CDeterministicMNList
      */
     [[nodiscard]] std::vector<CDeterministicMNCPtr> GetProjectedMNPayees(gsl::not_null<const CBlockIndex* const> pindexPrev, int nCount = std::numeric_limits<int>::max()) const;
 
+    /**
+     * Calculates CSimplifiedMNList for current list and cache it
+     * Thread safety: Uses internal mutex for thread-safe cache access
+     */
+    gsl::not_null<std::shared_ptr<const CSimplifiedMNList>> to_sml() const;
+
     /**
      * Calculates the maximum penalty which is allowed at the height of this MN list. It is dynamic and might change
      * for every block.