backport: bitcoin/bitcoin#24304: [kernel 0/n] Introduce dash-chainstate

.gitignore

@@ -13,6 +13,7 @@ src/dash-gui
 src/dash-node
 src/dash-tx
 src/dash-util
+src/dash-chainstate
 src/dash-wallet
 src/test/fuzz/fuzz
 src/test/test_dash

ci/dash/matrix.sh

@@ -27,7 +27,7 @@ elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then
 elif [ "$BUILD_TARGET" = "linux64_multiprocess" ]; then
   source ./ci/test/00_setup_env_native_multiprocess.sh
 elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then
-  source ./ci/test/00_setup_env_native_nowallet.sh
+  source ./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh
 elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then
   source ./ci/test/00_setup_env_native_sqlite.sh
 elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then

ci/test/00_setup_env_native_nowallet.sh → ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh

@@ -6,9 +6,9 @@
 
 export LC_ALL=C.UTF-8
 
-export CONTAINER_NAME=ci_native_nowallet
+export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel
 export HOST=x86_64-pc-linux-gnu
 export PACKAGES="python3-zmq"
 export DEP_OPTS="NO_WALLET=1 CC=gcc-14 CXX=g++-14"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-reduce-exports CC=gcc-14 CXX=g++-14"
+export BITCOIN_CONFIG="--enable-reduce-exports CC=gcc-14 CXX=g++-14 --enable-experimental-util-chainstate"

configure.ac

@@ -35,6 +35,7 @@ BITCOIN_TEST_NAME=test_dash
 BITCOIN_CLI_NAME=dash-cli
 BITCOIN_TX_NAME=dash-tx
 BITCOIN_UTIL_NAME=dash-util
+BITCOIN_CHAINSTATE_NAME=dash-chainstate
 BITCOIN_WALLET_TOOL_NAME=dash-wallet
 dnl Multi Process
 BITCOIN_MP_NODE_NAME=dash-node
@@ -769,6 +770,13 @@ AC_ARG_ENABLE([util-util],
   [build_bitcoin_util=$enableval],
   [build_bitcoin_util=$build_bitcoin_utils])
 
+AC_ARG_ENABLE([experimental-util-chainstate],
+  [AS_HELP_STRING([--enable-experimental-util-chainstate],
+  [build experimental dash-chainstate executable (default=no)])],
+  [build_bitcoin_chainstate=$enableval],
+  [build_bitcoin_chainstate=no])
+
+
 AC_ARG_WITH([libs],
   [AS_HELP_STRING([--with-libs],
   [build libraries (default=yes)])],
@@ -1443,6 +1451,7 @@ if test "$enable_fuzz" = "yes"; then
   build_bitcoin_cli=no
   build_bitcoin_tx=no
   build_bitcoin_util=no
+  build_bitcoin_chainstate=no
   build_bitcoin_wallet=no
   build_bitcoind=no
   build_bitcoin_libs=no
@@ -1781,6 +1790,10 @@ AC_MSG_CHECKING([whether to build dash-util])
 AM_CONDITIONAL([BUILD_BITCOIN_UTIL], [test $build_bitcoin_util = "yes"])
 AC_MSG_RESULT($build_bitcoin_util)
 
+AC_MSG_CHECKING([whether to build experimental dash-chainstate])
+AM_CONDITIONAL([BUILD_BITCOIN_CHAINSTATE], [test $build_bitcoin_chainstate = "yes"])
+AC_MSG_RESULT($build_bitcoin_chainstate)
+
 AC_MSG_CHECKING([whether to build libraries])
 AM_CONDITIONAL([BUILD_BITCOIN_LIBS], [test $build_bitcoin_libs = "yes"])
 if test "$build_bitcoin_libs" = "yes"; then
@@ -1995,6 +2008,7 @@ AC_SUBST(BITCOIN_TEST_NAME)
 AC_SUBST(BITCOIN_CLI_NAME)
 AC_SUBST(BITCOIN_TX_NAME)
 AC_SUBST(BITCOIN_UTIL_NAME)
+AC_SUBST(BITCOIN_CHAINSTATE_NAME)
 AC_SUBST(BITCOIN_WALLET_TOOL_NAME)
 AC_SUBST(BITCOIN_MP_NODE_NAME)
 AC_SUBST(BITCOIN_MP_GUI_NAME)

src/Makefile.am

@@ -148,6 +148,10 @@ if BUILD_BITCOIN_UTIL
   bin_PROGRAMS += dash-util
 endif
 
+if BUILD_BITCOIN_CHAINSTATE
+  bin_PROGRAMS += dash-chainstate
+endif
+
 .PHONY: FORCE check-symbols check-security
 # dash core #
 BITCOIN_CORE_H = \
@@ -1156,6 +1160,163 @@ dash_util_LDADD = \
   $(BACKTRACE_LIBS)
 
 dash_util_LDADD += $(BOOST_LIBS)
+
+# dash-chainstate binary #
+dash_chainstate_SOURCES = \
+  bitcoin-chainstate.cpp \
+  addressindex.cpp \
+  arith_uint256.cpp \
+  base58.cpp \
+  batchedlogger.cpp \
+  bech32.cpp \
+  blockfilter.cpp \
+  bls/bls.cpp \
+  bls/bls_worker.cpp \
+  chain.cpp \
+  chainparamsbase.cpp \
+  chainparams.cpp \
+  clientversion.cpp \
+  coins.cpp \
+  compressor.cpp \
+  chainlock/clsig.cpp \
+  chainlock/chainlock.cpp \
+  consensus/merkle.cpp \
+  consensus/tx_check.cpp \
+  consensus/tx_verify.cpp \
+  common/bloom.cpp \
+  core_read.cpp \
+  dbwrapper.cpp \
+  deploymentinfo.cpp \
+  deploymentstatus.cpp \
+  evo/assetlocktx.cpp \
+  evo/cbtx.cpp \
+  evo/creditpool.cpp \
+  evo/chainhelper.cpp \
+  evo/deterministicmns.cpp \
+  evo/dmnstate.cpp \
+  evo/evodb.cpp \
+  evo/netinfo.cpp \
+  evo/mnhftx.cpp \
+  evo/providertx.cpp \
+  evo/simplifiedmns.cpp \
+  evo/smldiff.cpp \
+  evo/specialtx.cpp \
+  evo/specialtx_filter.cpp \
+  evo/specialtxman.cpp \
+  flatfile.cpp \
+  fs.cpp \
+  governance/classes.cpp \
+  governance/common.cpp \
+  governance/exceptions.cpp \
+  governance/governance.cpp \
+  governance/object.cpp \
+  governance/validators.cpp \
+  governance/vote.cpp \
+  governance/votedb.cpp \
+  gsl/assert.cpp \
+  hash.cpp \
+  index/base.cpp \
+  index/blockfilterindex.cpp \
+  index/coinstatsindex.cpp \
+  index/txindex.cpp \
+  instantsend/db.cpp \
+  instantsend/instantsend.cpp \
+  init/common.cpp \
+  key.cpp \
+  key_io.cpp \
+  llmq/blockprocessor.cpp \
+  llmq/commitment.cpp \
+  llmq/context.cpp \
+  llmq/debug.cpp \
+  llmq/dkgsession.cpp \
+  llmq/dkgsessionhandler.cpp \
+  llmq/dkgsessionmgr.cpp \
+  llmq/options.cpp \
+  llmq/quorums.cpp \
+  llmq/quorumsman.cpp \
+  llmq/signhash.cpp \
+  llmq/signing.cpp \
+  llmq/snapshot.cpp \
+  llmq/utils.cpp \
+  logging.cpp \
+  netaddress.cpp \
+  netbase.cpp \
+  node/blockstorage.cpp \
+  node/chainstate.cpp \
+  node/transaction.cpp \
+  kernel/coinstats.cpp \
+  masternode/meta.cpp \
+  masternode/payments.cpp \
+  masternode/sync.cpp \
+  messagesigner.cpp \
+  merkleblock.cpp \
+  node/interface_ui.cpp \
+  policy/feerate.cpp \
+  policy/fees.cpp \
+  policy/packages.cpp \
+  policy/policy.cpp \
+  policy/settings.cpp \
+  pow.cpp \
+  protocol.cpp \
+  primitives/block.cpp \
+  primitives/transaction.cpp \
+  pubkey.cpp \
+  random.cpp \
+  randomenv.cpp \
+  saltedhasher.cpp \
+  scheduler.cpp \
+  script/interpreter.cpp \
+  script/script.cpp \
+  script/script_error.cpp \
+  script/sigcache.cpp \
+  script/standard.cpp \
+  spork.cpp \
+  shutdown.cpp \
+  support/cleanse.cpp \
+  support/lockedpool.cpp \
+  sync.cpp \
+  timedata.cpp \
+  txdb.cpp \
+  txmempool.cpp \
+  uint256.cpp \
+  util/asmap.cpp \
+  util/bytevectorhash.cpp \
+  util/check.cpp \
+  util/getuniquepath.cpp \
+  util/hasher.cpp \
+  util/message.cpp \
+  util/moneystr.cpp \
+  util/ranges_set.cpp \
+  util/serfloat.cpp \
+  util/settings.cpp \
+  util/sock.cpp \
+  util/string.cpp \
+  util/strencodings.cpp \
+  util/syserror.cpp \
+  util/system.cpp \
+  util/thread.cpp \
+  util/threadinterrupt.cpp \
+  util/threadnames.cpp \
+  util/time.cpp \
+  util/tokenpipe.cpp \
+  validation.cpp \
+  validationinterface.cpp \
+  versionbits.cpp \
+  warnings.cpp
+dash_chainstate_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS)
+dash_chainstate_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+dash_chainstate_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
+dash_chainstate_LDADD = \
+  $(LIBBITCOIN_CRYPTO) \
+  $(LIBUNIVALUE) \
+  $(LIBSECP256K1) \
+  $(LIBDASHBLS) \
+  $(LIBLEVELDB) \
+  $(LIBMEMENV)
+
+# Required for obj/build.h to be generated first.
+# More details: https://www.gnu.org/software/automake/manual/html_node/Built-Sources-Example.html
+dash_chainstate-clientversion.$(OBJEXT): obj/build.h
 #
 
 # dashconsensus library #

src/active/dkgsessionhandler.cpp

@@ -10,8 +10,10 @@
 #include <llmq/blockprocessor.h>
 #include <llmq/debug.h>
 #include <llmq/dkgsession.h>
+#include <llmq/net_quorum.h>
 #include <llmq/options.h>
 #include <llmq/utils.h>
+#include <masternode/meta.h>
 
 #include <deploymentstatus.h>
 #include <logging.h>
@@ -423,6 +425,50 @@ bool ProcessPendingMessageBatch(const CConnman& connman, CDKGSession& session, C
     return true;
 }
 
+static void AddQuorumProbeConnections(const Consensus::LLMQParams& llmqParams, CConnman& connman, CMasternodeMetaMan& mn_metaman,
+                               const CSporkManager& sporkman, const UtilParameters& util_params,
+                               const CDeterministicMNList& tip_mn_list, const uint256& myProTxHash)
+{
+    assert(mn_metaman.IsValid());
+
+    if (!IsQuorumPoseEnabled(llmqParams.type, sporkman)) {
+        return;
+    }
+
+    auto members = utils::GetAllQuorumMembers(llmqParams.type, util_params);
+    auto curTime = GetTime<std::chrono::seconds>().count();
+
+    Uint256HashSet probeConnections;
+    for (const auto& dmn : members) {
+        if (dmn->proTxHash == myProTxHash) {
+            continue;
+        }
+        auto lastOutbound = mn_metaman.GetLastOutboundSuccess(dmn->proTxHash);
+        if (curTime - lastOutbound < 10 * 60) {
+            // avoid re-probing nodes too often
+            continue;
+        }
+        probeConnections.emplace(dmn->proTxHash);
+    }
+
+    if (!probeConnections.empty()) {
+        if (LogAcceptDebug(BCLog::LLMQ)) {
+            std::string debugMsg = strprintf("%s -- adding masternodes probes for quorum %s:\n", __func__,
+                                             util_params.m_base_index->GetBlockHash().ToString());
+            for (const auto& c : probeConnections) {
+                auto dmn = tip_mn_list.GetValidMN(c);
+                if (!dmn) {
+                    debugMsg += strprintf("  %s (not in valid MN set anymore)\n", c.ToString());
+                } else {
+                    debugMsg += strprintf("  %s (%s)\n", c.ToString(),
+                                          dmn->pdmnState->netInfo->GetPrimary().ToStringAddrPort());
+                }
+            }
+            LogPrint(BCLog::NET_NETCONN, debugMsg.c_str()); /* Continued */
+        }
+        connman.AddPendingProbeConnections(probeConnections);
+    }
+}
 void ActiveDKGSessionHandler::HandleDKGRound(CConnman& connman, PeerManager& peerman)
 {
     WaitForNextPhase(std::nullopt, QuorumPhase::Initialized);
@@ -460,10 +506,10 @@ void ActiveDKGSessionHandler::HandleDKGRound(CConnman& connman, PeerManager& pee
     }
 
     const auto tip_mn_list = m_dmnman.GetListAtChainTip();
-    utils::EnsureQuorumConnections(params, connman, m_sporkman, {m_dmnman, m_qsnapman, m_chainman, pQuorumBaseBlockIndex},
+    llmq::EnsureQuorumConnections(params, connman, m_sporkman, {m_dmnman, m_qsnapman, m_chainman, pQuorumBaseBlockIndex},
                                    tip_mn_list, curSession->ProTx(), /*is_masternode=*/true, m_quorums_watch);
     if (curSession->AreWeMember()) {
-        utils::AddQuorumProbeConnections(params, connman, m_mn_metaman, m_sporkman,
+        AddQuorumProbeConnections(params, connman, m_mn_metaman, m_sporkman,
                                          {m_dmnman, m_qsnapman, m_chainman, pQuorumBaseBlockIndex}, tip_mn_list,
                                          curSession->ProTx());
     }

src/active/dkgsessionhandler.h

@@ -113,6 +113,7 @@ class ActiveDKGSessionHandler final : public llmq::CDKGSessionHandler
     void HandleDKGRound(CConnman& connman, PeerManager& peerman) EXCLUSIVE_LOCKS_REQUIRED(!cs_phase_qhash);
     void PhaseHandlerThread(CConnman& connman, PeerManager& peerman) EXCLUSIVE_LOCKS_REQUIRED(!cs_phase_qhash);
 };
+
 } // namespace llmq
 
 #endif // BITCOIN_ACTIVE_DKGSESSIONHANDLER_H