backport: merge bitcoin#21391, #19550, #15946, #21230, #21252, #21297, #20605, #21575, #22309, #19762 (assumeutxo backports: part 4)

doc/JSON-RPC-interface.md

@@ -5,6 +5,28 @@ The headless daemon `dashd` has the JSON-RPC API enabled by default, the GUI
 option. In the GUI it is possible to execute RPC methods in the Debug Console
 Dialog.
 
+## Parameter passing
+
+The JSON-RPC server supports both _by-position_ and _by-name_ [parameter
+structures](https://www.jsonrpc.org/specification#parameter_structures)
+described in the JSON-RPC specification. For extra convenience, to avoid the
+need to name every parameter value, all RPC methods accept a named parameter
+called `args`, which can be set to an array of initial positional values that
+are combined with named values.
+
+Examples:
+
+```sh
+# "params": ["mywallet", false, false, "", false, false, true]
+dash-cli createwallet mywallet false false "" false false true
+
+# "params": {"wallet_name": "mywallet", "load_on_startup": true}
+dash-cli -named createwallet wallet_name=mywallet load_on_startup=true
+
+# "params": {"args": ["mywallet"], "load_on_startup": true}
+dash-cli -named createwallet mywallet load_on_startup=true
+```
+
 ## Versioning
 
 The RPC interface might change from one major version of Dash Core to the

doc/release-notes-19550.md

@@ -0,0 +1,6 @@
+New RPCs
+--------
+
+- The `getindexinfo` RPC returns the actively running indices of the node,
+  including their current sync status and height. It also accepts an `index_name`
+  to specify returning only the status of that index.

doc/release-notes-19762.md

@@ -0,0 +1,19 @@
+JSON-RPC
+---
+
+All JSON-RPC methods accept a new [named
+parameter](JSON-RPC-interface.md#parameter-passing) called `args` that can
+contain positional parameter values. This is a convenience to allow some
+parameter values to be passed by name without having to name every value. The
+python test framework and `dash-cli` tool both take advantage of this, so
+for example:
+
+```sh
+dash-cli -named createwallet wallet_name=mywallet load_on_startup=1
+```
+
+Can now be shortened to:
+
+```sh
+dash-cli -named createwallet mywallet load_on_startup=1
+```

src/Makefile.am

@@ -247,6 +247,7 @@ BITCOIN_CORE_H = \
   netbase.h \
   netfulfilledman.h \
   netmessagemaker.h \
+  node/blockstorage.h \
   node/coin.h \
   node/coinstats.h \
   node/context.h \
@@ -441,6 +442,7 @@ libbitcoin_server_a_SOURCES = \
   net.cpp \
   netfulfilledman.cpp \
   net_processing.cpp \
+  node/blockstorage.cpp \
   node/coin.cpp \
   node/coinstats.cpp \
   node/context.cpp \

src/bitcoind.cpp

@@ -30,15 +30,6 @@
 
 const std::function<std::string(const char*)> G_TRANSLATION_FUN = nullptr;
 
-static void WaitForShutdown(NodeContext& node)
-{
-    while (!ShutdownRequested())
-    {
-        UninterruptibleSleep(std::chrono::milliseconds{200});
-    }
-    Interrupt(node);
-}
-
 //////////////////////////////////////////////////////////////////////////////
 //
 // Start
@@ -155,12 +146,10 @@ static bool AppInit(int argc, char* argv[])
         PrintExceptionContinue(std::current_exception(), "AppInit()");
     }
 
-    if (!fRet)
-    {
-        Interrupt(node);
-    } else {
-        WaitForShutdown(node);
+    if (fRet) {
+        WaitForShutdown();
     }
+    Interrupt(node);
     Shutdown(node);
 
     return fRet;

src/chainparams.cpp

@@ -780,7 +780,7 @@ class CRegTestParams : public CChainParams {
         nDefaultPort = 19899;
         nDefaultPlatformP2PPort = 22200;
         nDefaultPlatformHTTPPort = 22201;
-        nPruneAfterHeight = 1000;
+        nPruneAfterHeight = gArgs.GetBoolArg("-fastprune", false) ? 100 : 1000;
         m_assumed_blockchain_size = 0;
         m_assumed_chain_state_size = 0;
 

src/context.h

@@ -24,10 +24,10 @@ using CoreContext = std::variant<std::nullopt_t,
                                  std::reference_wrapper<LLMQContext>>;
 
 template<typename T>
-T* GetContext(const CoreContext& ctx) noexcept
+T* GetContext(const CoreContext& context) noexcept
 {
-    return std::holds_alternative<std::reference_wrapper<T>>(ctx)
-                ? &std::get<std::reference_wrapper<T>>(ctx).get()
+    return std::holds_alternative<std::reference_wrapper<T>>(context)
+                ? &std::get<std::reference_wrapper<T>>(context).get()
                 : nullptr;
 }
 

src/evo/cbtx.cpp

@@ -8,6 +8,7 @@
 #include <llmq/chainlocks.h>
 #include <llmq/commitment.h>
 #include <llmq/utils.h>
+#include <node/blockstorage.h>
 #include <evo/simplifiedmns.h>
 #include <evo/specialtx.h>
 #include <consensus/validation.h>

src/evo/creditpool.cpp

@@ -12,6 +12,7 @@
 #include <chain.h>
 #include <logging.h>
 #include <validation.h>
+#include <node/blockstorage.h>
 
 #include <algorithm>
 #include <exception>

src/evo/simplifiedmns.cpp

@@ -11,6 +11,7 @@
 #include <llmq/commitment.h>
 #include <llmq/quorums.h>
 #include <llmq/utils.h>
+#include <node/blockstorage.h>
 #include <evo/specialtx.h>
 
 #include <pubkey.h>

src/governance/governance.cpp

@@ -555,7 +555,7 @@ bool CGovernanceManager::ConfirmInventoryRequest(const CInv& inv)
         return false;
     }
 
-    const auto& [_, inserted] = setHash->insert(inv.hash);
+    const auto& [_itr, inserted] = setHash->insert(inv.hash);
 
     if (inserted) {
         LogPrint(BCLog::GOBJECT, "CGovernanceManager::ConfirmInventoryRequest added inv to requested set\n");

src/index/base.cpp

@@ -4,11 +4,12 @@
 
 #include <chainparams.h>
 #include <index/base.h>
+#include <node/blockstorage.h>
 #include <shutdown.h>
 #include <tinyformat.h>
 #include <ui_interface.h>
 #include <util/translation.h>
-#include <validation.h>
+#include <validation.h> // For g_chainman
 #include <warnings.h>
 
 constexpr char DB_BEST_BLOCK = 'B';
@@ -64,6 +65,43 @@ bool BaseIndex::Init()
         m_best_block_index = g_chainman.m_blockman.FindForkInGlobalIndex(::ChainActive(), locator);
     }
     m_synced = m_best_block_index.load() == ::ChainActive().Tip();
+    if (!m_synced) {
+        bool prune_violation = false;
+        if (!m_best_block_index) {
+            // index is not built yet
+            // make sure we have all block data back to the genesis
+            const CBlockIndex* block = ::ChainActive().Tip();
+            while (block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
+                block = block->pprev;
+            }
+            prune_violation = block != ::ChainActive().Genesis();
+        }
+        // in case the index has a best block set and is not fully synced
+        // check if we have the required blocks to continue building the index
+        else {
+            const CBlockIndex* block_to_test = m_best_block_index.load();
+            if (!ChainActive().Contains(block_to_test)) {
+                // if the bestblock is not part of the mainchain, find the fork
+                // and make sure we have all data down to the fork
+                block_to_test = ::ChainActive().FindFork(block_to_test);
+            }
+            const CBlockIndex* block = ::ChainActive().Tip();
+            prune_violation = true;
+            // check backwards from the tip if we have all block data until we reach the indexes bestblock
+            while (block_to_test && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
+                if (block_to_test == block) {
+                    prune_violation = false;
+                    break;
+                }
+                block = block->pprev;
+            }
+        }
+        if (prune_violation) {
+            // throw error and graceful shutdown if we can't build the index
+            FatalError("%s: %s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)", __func__, GetName());
+            return false;
+        }
+    }
     return true;
 }
 
@@ -176,6 +214,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti
     assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
 
     // In the case of a reorg, ensure persisted block locator is not stale.
+    // Pruning has a minimum of 288 blocks-to-keep and getting the index
+    // out of sync may be possible but a users fault.
+    // In case we reorg beyond the pruned depth, ReadBlockFromDisk would
+    // throw and lead to a graceful shutdown
     m_best_block_index = new_tip;
     if (!Commit()) {
         // If commit fails, revert the best block index to avoid corruption.
@@ -318,3 +360,12 @@ void BaseIndex::Stop()
         m_thread_sync.join();
     }
 }
+
+IndexSummary BaseIndex::GetSummary() const
+{
+    IndexSummary summary{};
+    summary.name = GetName();
+    summary.synced = m_synced;
+    summary.best_block_height = m_best_block_index ? m_best_block_index.load()->nHeight : 0;
+    return summary;
+}

src/index/base.h

@@ -13,6 +13,12 @@
 
 class CBlockIndex;
 
+struct IndexSummary {
+    std::string name;
+    bool synced{false};
+    int best_block_height{0};
+};
+
 /**
  * Base class for indices of blockchain data. This implements
  * CValidationInterface and ensures blocks are indexed sequentially according
@@ -113,6 +119,9 @@ class BaseIndex : public CValidationInterface
 
     /// Stops the instance from staying in sync with blockchain updates.
     void Stop();
+
+    /// Get a summary of the index and its state.
+    IndexSummary GetSummary() const;
 };
 
 #endif // BITCOIN_INDEX_BASE_H

src/index/blockfilterindex.cpp

@@ -6,9 +6,8 @@
 
 #include <dbwrapper.h>
 #include <index/blockfilterindex.h>
+#include <node/blockstorage.h>
 #include <serialize.h>
-#include <util/system.h>
-#include <validation.h>
 
 /* The index database stores three items for each block: the disk location of the encoded filter,
  * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by