From 8fef5440022df1a892655ab8da6b9792511f729b Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 4 Apr 2018 11:56:22 -0700 Subject: [PATCH 01/40] Add Slice: a (pointer, size) array view that acts like a container --- src/Makefile.am | 1 + src/span.h | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 src/span.h diff --git a/src/Makefile.am b/src/Makefile.am index a747e373eaa0..08ced54a504e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -262,6 +262,7 @@ BITCOIN_CORE_H = \ script/standard.h \ script/script_error.h \ serialize.h \ + span.h \ spork.h \ sporkdb.h \ sporkid.h \ diff --git a/src/span.h b/src/span.h new file mode 100644 index 000000000000..707fc2191869 --- /dev/null +++ b/src/span.h @@ -0,0 +1,40 @@ +// Copyright (c) 2018 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SPAN_H +#define BITCOIN_SPAN_H + +#include +#include + +/** A Span is an object that can refer to a contiguous sequence of objects. + * + * It implements a subset of C++20's std::span. + */ +template +class Span +{ + C* m_data; + std::ptrdiff_t m_size; + +public: + constexpr Span() noexcept : m_data(nullptr), m_size(0) {} + constexpr Span(C* data, std::ptrdiff_t size) noexcept : m_data(data), m_size(size) {} + + constexpr C* data() const noexcept { return m_data; } + constexpr std::ptrdiff_t size() const noexcept { return m_size; } +}; + +/** Create a span to a container exposing data() and size(). + * + * This correctly deals with constness: the returned Span's element type will be + * whatever data() returns a pointer to. If either the passed container is const, + * or its element type is const, the resulting span will have a const element type. + * + * std::span will have a constructor that implements this functionality directly. + */ +template +constexpr Span().data())>::type> MakeSpan(V& v) { return Span().data())>::type>(v.data(), v.size()); } + +#endif From 1ef2d90be84feedabc36a83b433af51a6c11ef24 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 4 Apr 2018 12:40:10 -0700 Subject: [PATCH 02/40] Support serializing Span and use that instead of FLATDATA --- src/compressor.h | 9 +++++---- src/netaddress.h | 8 ++++++-- src/serialize.h | 48 +++++------------------------------------------- 3 files changed, 16 insertions(+), 49 deletions(-) diff --git a/src/compressor.h b/src/compressor.h index 65ea6be0dd76..a2ced07df35d 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -10,6 +10,7 @@ #include "primitives/transaction.h" #include "script/script.h" #include "serialize.h" +#include "span.h" class CKeyID; class CPubKey; @@ -63,12 +64,12 @@ class CScriptCompressor { std::vector compr; if (Compress(compr)) { - s << CFlatData(compr); + s << MakeSpan(compr); return; } unsigned int nSize = script.size() + nSpecialScripts; s << VARINT(nSize); - s << CFlatData(script); + s << MakeSpan(script); } template @@ -78,7 +79,7 @@ class CScriptCompressor s >> VARINT(nSize); if (nSize < nSpecialScripts) { std::vector vch(GetSpecialSize(nSize), 0x00); - s >> CFlatData(vch); + s >> MakeSpan(vch); Decompress(nSize, vch); return; } @@ -89,7 +90,7 @@ class CScriptCompressor s.ignore(nSize); } else { script.resize(nSize); - s >> CFlatData(script); + s >> MakeSpan(script); } } }; diff --git a/src/netaddress.h b/src/netaddress.h index e5f3ef6e19a5..f3dfc6fc1fd7 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -12,6 +12,7 @@ #include "compat.h" #include "serialize.h" +#include "span.h" #include #include @@ -171,10 +172,13 @@ class CService : public CNetAddr inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(ip); + + // TODO: introduce native support for BE serialization in serialize.h unsigned short portN = htons(port); - READWRITE(FLATDATA(portN)); - if (ser_action.ForRead()) + READWRITE(Span((unsigned char*)&portN, 2)); + if (ser_action.ForRead()) { port = ntohs(portN); + } } }; diff --git a/src/serialize.h b/src/serialize.h index dde4883be959..5fc811a2b6c5 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -27,6 +27,7 @@ #include "libzerocoin/SpendType.h" #include "optional.h" #include "prevector.h" +#include "span.h" #include "sporkid.h" class CScript; @@ -49,7 +50,7 @@ constexpr deserialize_type deserialize {}; /** * Used to bypass the rule against non-const reference to temporary - * where it makes sense with wrappers such as CFlatData or CTxDB + * where it makes sense with wrappers. */ template inline T& REF(const T& val) @@ -203,6 +204,8 @@ template inline void Serialize(Stream& s, float a ) { ser_wri template inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); } template inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); } template inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); } +template inline void Serialize(Stream& s, const Span& span) { s.write(CharCast(span.data()), span.size()); } +template inline void Serialize(Stream& s, const Span& span) { s.write(CharCast(span.data()), span.size()); } template inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char template inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); } @@ -217,6 +220,7 @@ template inline void Unserialize(Stream& s, float& a ) { a = template inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); } template inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); } template inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); } +template inline void Unserialize(Stream& s, Span& span) { s.read(CharCast(span.data()), span.size()); } template inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); } template inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; } @@ -433,52 +437,10 @@ I ReadVarInt(Stream& is) } } -#define FLATDATA(obj) CFlatData((char*)&(obj), (char*)&(obj) + sizeof(obj)) #define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj)) #define COMPACTSIZE(obj) CCompactSize(REF(obj)) #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) -/** - * Wrapper for serializing arrays and POD. - */ -class CFlatData -{ -protected: - char* pbegin; - char* pend; - -public: - CFlatData(void* pbeginIn, void* pendIn) : pbegin((char*)pbeginIn), pend((char*)pendIn) {} - template - explicit CFlatData(std::vector& v) - { - pbegin = (char*)v.data(); - pend = (char*)(v.data() + v.size()); - } - template - explicit CFlatData(prevector &v) - { - pbegin = (char*)v.data(); - pend = (char*)(v.data() + v.size()); - } - char* begin() { return pbegin; } - const char* begin() const { return pbegin; } - char* end() { return pend; } - const char* end() const { return pend; } - - template - void Serialize(Stream& s) const - { - s.write(pbegin, pend - pbegin); - } - - template - void Unserialize(Stream& s) - { - s.read(pbegin, pend - pbegin); - } -}; - template class CVarInt { From fb3c64685f26cb0f77120b1e43bbe5e159237f10 Mon Sep 17 00:00:00 2001 From: furszy Date: Sun, 30 May 2021 21:56:05 -0300 Subject: [PATCH 03/40] Migrate last FLATDATA calls to use Span. --- src/addrdb.cpp | 2 +- src/budget/budgetdb.cpp | 7 +++---- src/masternode-payments.cpp | 8 +++----- src/masternodeman.cpp | 6 +++--- src/wallet/wallet.h | 4 ++-- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 318bd7e96374..3c23890d3637 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -83,7 +83,7 @@ bool DeserializeDB(Stream& stream, Data& data, bool fCheckSum = true) unsigned char pchMsgTmp[4]; verifier >> pchMsgTmp; // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) + if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp)) != 0) return error("%s: Invalid network magic number", __func__); // de-serialize data diff --git a/src/budget/budgetdb.cpp b/src/budget/budgetdb.cpp index b50e6c93d6db..069a6a3a0943 100644 --- a/src/budget/budgetdb.cpp +++ b/src/budget/budgetdb.cpp @@ -93,7 +93,6 @@ CBudgetDB::ReadResult CBudgetDB::Read(CBudgetManager& objToLoad, bool fDryRun) } int version; - unsigned char pchMsgTmp[4]; std::string strMagicMessageTmp; try { // de-serialize file header @@ -106,12 +105,12 @@ CBudgetDB::ReadResult CBudgetDB::Read(CBudgetManager& objToLoad, bool fDryRun) return IncorrectMagicMessage; } - // de-serialize file header (network specific magic number) and .. - ssObj >> FLATDATA(pchMsgTmp); + std::vector pchMsgTmp(4); + ssObj >> MakeSpan(pchMsgTmp); // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + if (memcmp(pchMsgTmp.data(), Params().MessageStart(), pchMsgTmp.size()) != 0) { error("%s : Invalid network magic number", __func__); return IncorrectMagicNumber; } diff --git a/src/masternode-payments.cpp b/src/masternode-payments.cpp index 18069f12999b..8497bac5673c 100644 --- a/src/masternode-payments.cpp +++ b/src/masternode-payments.cpp @@ -5,7 +5,6 @@ #include "masternode-payments.h" -#include "addrman.h" #include "chainparams.h" #include "evo/deterministicmns.h" #include "fs.h" @@ -112,7 +111,6 @@ CMasternodePaymentDB::ReadResult CMasternodePaymentDB::Read(CMasternodePayments& } int version; - unsigned char pchMsgTmp[4]; std::string strMagicMessageTmp; try { // de-serialize file header @@ -125,12 +123,12 @@ CMasternodePaymentDB::ReadResult CMasternodePaymentDB::Read(CMasternodePayments& return IncorrectMagicMessage; } - // de-serialize file header (network specific magic number) and .. - ssObj >> FLATDATA(pchMsgTmp); + std::vector pchMsgTmp(4); + ssObj >> MakeSpan(pchMsgTmp); // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + if (memcmp(pchMsgTmp.data(), Params().MessageStart(), pchMsgTmp.size()) != 0) { error("%s : Invalid network magic number", __func__); return IncorrectMagicNumber; } diff --git a/src/masternodeman.cpp b/src/masternodeman.cpp index bf954a92c49e..d7bb4ca1e73a 100644 --- a/src/masternodeman.cpp +++ b/src/masternodeman.cpp @@ -123,7 +123,6 @@ CMasternodeDB::ReadResult CMasternodeDB::Read(CMasternodeMan& mnodemanToLoad) } int version; - unsigned char pchMsgTmp[4]; std::string strMagicMessageTmp; try { // de-serialize file header @@ -137,10 +136,11 @@ CMasternodeDB::ReadResult CMasternodeDB::Read(CMasternodeMan& mnodemanToLoad) } // de-serialize file header (network specific magic number) and .. - ssMasternodes >> FLATDATA(pchMsgTmp); + std::vector pchMsgTmp(4); + ssMasternodes >> MakeSpan(pchMsgTmp); // ... verify the network matches ours - if (memcmp(pchMsgTmp, Params().MessageStart(), sizeof(pchMsgTmp))) { + if (memcmp(pchMsgTmp.data(), Params().MessageStart(), pchMsgTmp.size()) != 0) { error("%s : Invalid network magic number", __func__); return IncorrectMagicNumber; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 5b3c9fd6a22d..a05c7062bd08 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -153,7 +153,7 @@ class CKeyPool READWRITE(vchPubKey); if (ser_action.ForRead()) { try { - READWRITE(FLATDATA(type)); + READWRITE(Span((unsigned char*)&type, 1)); READWRITE(m_pre_split); } catch (std::ios_base::failure&) { /* Set as external address if we can't read the type boolean @@ -162,7 +162,7 @@ class CKeyPool m_pre_split = true; } } else { - READWRITE(FLATDATA(type)); + READWRITE(Span((unsigned char*)&type, 1)); READWRITE(m_pre_split); } } From 5c36b3da257b32ee90593a5df70171fd5fc7a0f4 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 8 Apr 2018 09:39:05 -0700 Subject: [PATCH 04/40] Introduce BigEndian wrapper and use it for netaddress ports --- src/netaddress.h | 10 ++-------- src/serialize.h | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/netaddress.h b/src/netaddress.h index f3dfc6fc1fd7..c6133eb90394 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -143,7 +143,7 @@ class CSubNet class CService : public CNetAddr { protected: - unsigned short port; // host order + uint16_t port; // host order public: CService(); @@ -172,13 +172,7 @@ class CService : public CNetAddr inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(ip); - - // TODO: introduce native support for BE serialization in serialize.h - unsigned short portN = htons(port); - READWRITE(Span((unsigned char*)&portN, 2)); - if (ser_action.ForRead()) { - port = ntohs(portN); - } + READWRITE(WrapBigEndian(port)); } }; diff --git a/src/serialize.h b/src/serialize.h index 5fc811a2b6c5..94c00b28f1ee 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -87,6 +87,11 @@ template inline void ser_writedata16(Stream &s, uint16_t obj) obj = htole16(obj); s.write((char*)&obj, 2); } +template inline void ser_writedata16be(Stream &s, uint16_t obj) +{ + obj = htobe16(obj); + s.write((char*)&obj, 2); +} template inline void ser_writedata32(Stream &s, uint32_t obj) { obj = htole32(obj); @@ -109,6 +114,12 @@ template inline uint16_t ser_readdata16(Stream &s) s.read((char*)&obj, 2); return le16toh(obj); } +template inline uint16_t ser_readdata16be(Stream &s) +{ + uint16_t obj; + s.read((char*)&obj, 2); + return be16toh(obj); +} template inline uint32_t ser_readdata32(Stream &s) { uint32_t obj; @@ -461,6 +472,40 @@ class CVarInt } }; +/** Serialization wrapper class for big-endian integers. + * + * Use this wrapper around integer types that are stored in memory in native + * byte order, but serialized in big endian notation. This is only intended + * to implement serializers that are compatible with existing formats, and + * its use is not recommended for new data structures. + * + * Only 16-bit types are supported for now. + */ +template +class BigEndian +{ +protected: + I& m_val; +public: + explicit BigEndian(I& val) : m_val(val) + { + static_assert(std::is_unsigned::value, "BigEndian type must be unsigned integer"); + static_assert(sizeof(I) == 2 && std::numeric_limits::min() == 0 && std::numeric_limits::max() == std::numeric_limits::max(), "Unsupported BigEndian size"); + } + + template + void Serialize(Stream& s) const + { + ser_writedata16be(s, m_val); + } + + template + void Unserialize(Stream& s) + { + m_val = ser_readdata16be(s); + } +}; + class CCompactSize { protected: @@ -512,6 +557,9 @@ class LimitedString template CVarInt WrapVarInt(I& n) { return CVarInt{n}; } +template +BigEndian WrapBigEndian(I& n) { return BigEndian(n); } + /** * Forward declarations */ From f05e6922ada12986cfed48db8304792be5eec9aa Mon Sep 17 00:00:00 2001 From: furszy Date: Mon, 31 May 2021 00:38:44 -0300 Subject: [PATCH 05/40] Drop unused GetType() from CSizeComputer Based on btc@da74db0940720407fafaf3582bbaf9c81a4d3b4d --- src/coins.cpp | 2 +- src/core_write.cpp | 2 +- src/dbwrapper.h | 2 +- src/miner.cpp | 2 +- src/policy/policy.cpp | 2 +- src/primitives/transaction.cpp | 4 +-- src/qt/walletmodeltransaction.cpp | 2 +- src/rpc/blockchain.cpp | 4 +-- src/sapling/sapling_operation.cpp | 2 +- src/script/bitcoinconsensus.cpp | 2 +- src/serialize.h | 47 +++++++++++------------------- src/test/arith_uint256_tests.cpp | 8 ++--- src/test/mempool_tests.cpp | 10 +++---- src/test/policyestimator_tests.cpp | 2 +- src/test/serialize_tests.cpp | 6 ++-- src/txmempool.cpp | 2 +- src/undo.h | 2 +- src/validation.cpp | 12 ++++---- src/wallet/wallet.cpp | 8 ++--- 19 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index 8d4878ddeea0..725f4f050130 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -429,7 +429,7 @@ bool CCoinsViewCache::PruneInvalidEntries() return Flush(); } -static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_SIZE_CURRENT / ::GetSerializeSize(CTxOut(), SER_NETWORK, PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. +static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_SIZE_CURRENT / ::GetSerializeSize(CTxOut(), PROTOCOL_VERSION); // TODO: merge with similar definition in undo.h. const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) { diff --git a/src/core_write.cpp b/src/core_write.cpp index 5f0588c82fc1..d8ee0d37b146 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -167,7 +167,7 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry) entry.pushKV("txid", tx.GetHash().GetHex()); entry.pushKV("version", tx.nVersion); entry.pushKV("type", tx.nType); - entry.pushKV("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + entry.pushKV("size", (int)::GetSerializeSize(tx, PROTOCOL_VERSION)); entry.pushKV("locktime", (int64_t)tx.nLockTime); UniValue vin(UniValue::VARR); diff --git a/src/dbwrapper.h b/src/dbwrapper.h index dd0a48c24b28..033762b8201c 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -607,7 +607,7 @@ class CDBTransaction { template void Write(const CDataStream& ssKey, const V& v) { - auto valueMemoryUsage = ::GetSerializeSize(v, SER_DISK, CLIENT_VERSION); + auto valueMemoryUsage = ::GetSerializeSize(v, CLIENT_VERSION); if (deletes.erase(ssKey)) { memoryUsage -= ssKey.size(); } diff --git a/src/miner.cpp b/src/miner.cpp index 5277d2e9d2b4..1173b7c3dbc3 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -188,7 +188,7 @@ void BitcoinMiner(CWallet* pwallet, bool fProofOfStake) IncrementExtraNonce(pblock, pindexPrev->nHeight + 1, nExtraNonce); LogPrintf("Running PIVXMiner with %u transactions in block (%u bytes)\n", pblock->vtx.size(), - ::GetSerializeSize(*pblock, SER_NETWORK, PROTOCOL_VERSION)); + ::GetSerializeSize(*pblock, PROTOCOL_VERSION)); // // Search diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index c2dc818175ee..906ec817a6ac 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -31,7 +31,7 @@ CAmount GetDustThreshold(const CTxOut& txout, const CFeeRate& dustRelayFeeIn) if (txout.scriptPubKey.IsUnspendable()) return 0; - size_t nSize = GetSerializeSize(txout, SER_DISK, 0); + size_t nSize = GetSerializeSize(txout, 0); nSize += (32 + 4 + 1 + 107 + 4); // the 148 mentioned above return dustRelayFeeIn.GetFee(nSize); } diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 30defdbfacbd..be7a4fb3e8de 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -252,7 +252,7 @@ unsigned int CTransaction::CalculateModifiedSize(unsigned int nTxSize) const // Providing any more cleanup incentive than making additional inputs free would // risk encouraging people to create junk outputs to redeem later. if (nTxSize == 0) - nTxSize = ::GetSerializeSize(*this, SER_NETWORK, PROTOCOL_VERSION); + nTxSize = ::GetSerializeSize(*this, PROTOCOL_VERSION); for (std::vector::const_iterator it(vin.begin()); it != vin.end(); ++it) { unsigned int offset = 41U + std::min(110U, (unsigned int)it->scriptSig.size()); @@ -264,7 +264,7 @@ unsigned int CTransaction::CalculateModifiedSize(unsigned int nTxSize) const unsigned int CTransaction::GetTotalSize() const { - return ::GetSerializeSize(*this, SER_NETWORK, PROTOCOL_VERSION); + return ::GetSerializeSize(*this, PROTOCOL_VERSION); } std::string CTransaction::ToString() const diff --git a/src/qt/walletmodeltransaction.cpp b/src/qt/walletmodeltransaction.cpp index 709b100c6050..62cd2da3e915 100644 --- a/src/qt/walletmodeltransaction.cpp +++ b/src/qt/walletmodeltransaction.cpp @@ -30,7 +30,7 @@ CTransactionRef& WalletModelTransaction::getTransaction() unsigned int WalletModelTransaction::getTransactionSize() { - return (!walletTransaction ? 0 : (::GetSerializeSize(*walletTransaction, SER_NETWORK, PROTOCOL_VERSION))); + return (!walletTransaction ? 0 : (::GetSerializeSize(*walletTransaction, PROTOCOL_VERSION))); } CAmount WalletModelTransaction::getTransactionFee() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 08fbb73efc62..bff9a2f6a709 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -137,7 +137,7 @@ UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool tx if (chainActive.Contains(blockindex)) confirmations = chainActive.Height() - blockindex->nHeight + 1; result.pushKV("confirmations", confirmations); - result.pushKV("size", (int)::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION)); + result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); result.pushKV("height", blockindex->nHeight); result.pushKV("version", block.nVersion); result.pushKV("merkleroot", block.hashMerkleRoot.GetHex()); @@ -1361,7 +1361,7 @@ UniValue getblockindexstats(const JSONRPCRequest& request) { continue; // Transaction size - nBytes += GetSerializeSize(tx, SER_NETWORK, CLIENT_VERSION); + nBytes += GetSerializeSize(tx, CLIENT_VERSION); // Transparent inputs for (unsigned int j = 0; j < tx.vin.size(); j++) { diff --git a/src/sapling/sapling_operation.cpp b/src/sapling/sapling_operation.cpp index 0b9b5e5e9b09..24a2d17d320b 100644 --- a/src/sapling/sapling_operation.cpp +++ b/src/sapling/sapling_operation.cpp @@ -569,7 +569,7 @@ OperationResult CheckTransactionSize(std::vector& recipients, } } CTransaction tx(mtx); - size_t txsize = GetSerializeSize(tx, SER_NETWORK, tx.nVersion) + CTXOUT_REGULAR_SIZE * nTransparentOuts; + size_t txsize = tx.GetTotalSize() + CTXOUT_REGULAR_SIZE * nTransparentOuts; if (fromTaddr) { txsize += CTXIN_SPEND_DUST_SIZE; txsize += CTXOUT_REGULAR_SIZE; // There will probably be taddr change diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 0f7349871ab3..6f0595886bb1 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -81,7 +81,7 @@ int bitcoinconsensus_verify_script(const unsigned char *scriptPubKey, unsigned i CTransaction tx(deserialize, stream); if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); - if (GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) != txToLen) + if (GetSerializeSize(tx, PROTOCOL_VERSION) != txToLen) return set_error(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); // Regardless of the verification result, the tx did not error. diff --git a/src/serialize.h b/src/serialize.h index 94c00b28f1ee..1fb864962d81 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -603,7 +603,6 @@ template void Unserialize(Stream& is * optional */ template unsigned int GetSerializeSize(const Optional &item); -template unsigned int GetSerializeSizeNetwork(const Optional &item); template void Serialize(Stream& os, const Optional& item); template void Unserialize(Stream& is, Optional& item); @@ -620,29 +619,6 @@ unsigned int GetSerializeSize(const std::array &item) return size; } -/** - * optional - */ -template -unsigned int GetSerializeSize(const Optional &item) -{ - if (item) { - return 1 + GetSerializeSize(*item); - } else { - return 1; - } -} - -template -unsigned int GetSerializeSizeNetwork(const Optional &item) -{ - if (item) { - return 1 + GetSerializeSize(*item, SER_NETWORK, 0); - } else { - return 1; - } -} - template void Serialize(Stream& os, const Optional& item) { @@ -1074,11 +1050,10 @@ class CSizeComputer protected: size_t nSize; - const int nType; const int nVersion; public: - CSizeComputer(int nTypeIn, int nVersionIn) : nSize(0), nType(nTypeIn), nVersion(nVersionIn) {} + explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {} void write(const char* psz, size_t _nSize) { @@ -1104,7 +1079,6 @@ class CSizeComputer } int GetVersion() const { return nVersion; } - int GetType() const { return nType; } }; template @@ -1155,15 +1129,28 @@ inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) } template -size_t GetSerializeSize(const T& t, int nType, int nVersion = 0) +size_t GetSerializeSize(const T& t, int nVersion = 0) { - return (CSizeComputer(nType, nVersion) << t).size(); + return (CSizeComputer(nVersion) << t).size(); } template size_t GetSerializeSize(const S& s, const T& t) { - return (CSizeComputer(s.GetType(), s.GetVersion()) << t).size(); + return (CSizeComputer(s.GetVersion()) << t).size(); +} + +/** + * optional + */ +template +unsigned int GetSerializeSize(const Optional &item) +{ + if (item) { + return 1 + GetSerializeSize(*item); + } else { + return 1; + } } #endif // PIVX_SERIALIZE_H diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index b56397a993a8..cfc4d01bcdf6 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -597,8 +597,8 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex begin() end() size() GetLow64 G BOOST_CHECK(R1L.GetLow64() == R1LLow64); BOOST_CHECK(HalfL.GetLow64() ==0x0000000000000000ULL); BOOST_CHECK(OneL.GetLow64() ==0x0000000000000001ULL); - BOOST_CHECK(GetSerializeSize(R1L, 0, PROTOCOL_VERSION) == 32); - BOOST_CHECK(GetSerializeSize(ZeroL, 0,PROTOCOL_VERSION) == 32); + BOOST_CHECK(GetSerializeSize(R1L, PROTOCOL_VERSION) == 32); + BOOST_CHECK(GetSerializeSize(ZeroL, PROTOCOL_VERSION) == 32); CDataStream ss(0, PROTOCOL_VERSION); ss << R1L; @@ -645,8 +645,8 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex begin() end() size() GetLow64 G BOOST_CHECK(R1S.GetLow64() == R1LLow64); BOOST_CHECK(HalfS.GetLow64() ==0x0000000000000000ULL); BOOST_CHECK(OneS.GetLow64() ==0x0000000000000001ULL); - BOOST_CHECK(GetSerializeSize(R1S, 0, PROTOCOL_VERSION) == 20); - BOOST_CHECK(GetSerializeSize(ZeroS, 0, PROTOCOL_VERSION) == 20); + BOOST_CHECK(GetSerializeSize(R1S, PROTOCOL_VERSION) == 20); + BOOST_CHECK(GetSerializeSize(ZeroS, PROTOCOL_VERSION) == 20); ss << R1S; BOOST_CHECK(ss.str() == std::string(R1Array,R1Array+20)); diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index da62536d506f..302fc54e1b2d 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -336,7 +336,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx2.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx2.vout[0].nValue = 2 * COIN; pool.addUnchecked(tx2.GetHash(), entry.Fee(20000LL).Priority(9.0).FromTx(tx2)); - uint64_t tx2Size = ::GetSerializeSize(tx2, SER_NETWORK, PROTOCOL_VERSION); + uint64_t tx2Size = ::GetSerializeSize(tx2, PROTOCOL_VERSION); /* lowest fee */ CMutableTransaction tx3 = CMutableTransaction(); @@ -384,7 +384,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx6.vout.resize(1); tx6.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx6.vout[0].nValue = 20 * COIN; - uint64_t tx6Size = ::GetSerializeSize(tx6, SER_NETWORK, PROTOCOL_VERSION); + uint64_t tx6Size = ::GetSerializeSize(tx6, PROTOCOL_VERSION); pool.addUnchecked(tx6.GetHash(), entry.Fee(0LL).FromTx(tx6)); BOOST_CHECK_EQUAL(pool.size(), 6); @@ -403,7 +403,7 @@ BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) tx7.vout.resize(1); tx7.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx7.vout[0].nValue = 10 * COIN; - uint64_t tx7Size = ::GetSerializeSize(tx7, SER_NETWORK, PROTOCOL_VERSION); + uint64_t tx7Size = ::GetSerializeSize(tx7, PROTOCOL_VERSION); /* set the fee to just below tx2's feerate when including ancestor */ CAmount fee = (20000/tx2Size)*(tx7Size + tx6Size) - 1; @@ -475,12 +475,12 @@ BOOST_AUTO_TEST_CASE(MempoolSizeLimitTest) BOOST_CHECK(pool.exists(tx2.GetHash())); BOOST_CHECK(pool.exists(tx3.GetHash())); - pool.TrimToSize(::GetSerializeSize(CTransaction(tx1), SER_NETWORK, PROTOCOL_VERSION)); // mempool is limited to tx1's size in memory usage, so nothing fits + pool.TrimToSize(::GetSerializeSize(CTransaction(tx1), PROTOCOL_VERSION)); // mempool is limited to tx1's size in memory usage, so nothing fits BOOST_CHECK(!pool.exists(tx1.GetHash())); BOOST_CHECK(!pool.exists(tx2.GetHash())); BOOST_CHECK(!pool.exists(tx3.GetHash())); - CFeeRate maxFeeRateRemoved(25000, ::GetSerializeSize(CTransaction(tx3), SER_NETWORK, PROTOCOL_VERSION) + ::GetSerializeSize(CTransaction(tx2), SER_NETWORK, PROTOCOL_VERSION)); + CFeeRate maxFeeRateRemoved(25000, ::GetSerializeSize(CTransaction(tx3), PROTOCOL_VERSION) + ::GetSerializeSize(CTransaction(tx2), PROTOCOL_VERSION)); BOOST_CHECK_EQUAL(pool.GetMinFee(1).GetFeePerK(), maxFeeRateRemoved.GetFeePerK() + 1000); CMutableTransaction tx4 = CMutableTransaction(); diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 91f672d52d81..d56ec3193213 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -42,7 +42,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) tx.vin[0].scriptSig = garbage; tx.vout.resize(1); tx.vout[0].nValue=0LL; - CFeeRate baseRate(basefee, ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION)); + CFeeRate baseRate(basefee, ::GetSerializeSize(tx, PROTOCOL_VERSION)); // Create a fake block std::vector block; diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 5a175ad7f18b..c17d51624077 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -19,7 +19,7 @@ void check_ser_rep(T thing, std::vector expected) CDataStream ss(SER_DISK, 0); ss << thing; - BOOST_CHECK(GetSerializeSize(thing, 0, 0) == ss.size()); + BOOST_CHECK(GetSerializeSize(thing, 0) == ss.size()); std::vector serialized_representation(ss.begin(), ss.end()); @@ -208,13 +208,13 @@ BOOST_AUTO_TEST_CASE(varints) CDataStream::size_type size = 0; for (int i = 0; i < 100000; i++) { ss << VARINT(i, VarIntMode::NONNEGATIVE_SIGNED); - size += ::GetSerializeSize(VARINT(i, VarIntMode::NONNEGATIVE_SIGNED), 0, 0); + size += ::GetSerializeSize(VARINT(i, VarIntMode::NONNEGATIVE_SIGNED), 0); BOOST_CHECK(size == ss.size()); } for (uint64_t i = 0; i < 100000000000ULL; i += 999999937) { ss << VARINT(i); - size += ::GetSerializeSize(VARINT(i), 0, 0); + size += ::GetSerializeSize(VARINT(i), 0); BOOST_CHECK(size == ss.size()); } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index ba2e95744463..147656939c24 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -28,7 +28,7 @@ CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& _tx, const CAmount& _nFe bool _spendsCoinbaseOrCoinstake, unsigned int _sigOps) : tx(MakeTransactionRef(_tx)), nFee(_nFee), nTime(_nTime), entryPriority(_entryPriority), entryHeight(_entryHeight), hadNoDependencies(poolHasNoInputsOf), inChainInputValue(_inChainInputValue), spendsCoinbaseOrCoinstake(_spendsCoinbaseOrCoinstake), sigOpCount(_sigOps) { - nTxSize = ::GetSerializeSize(*_tx, SER_NETWORK, PROTOCOL_VERSION); + nTxSize = ::GetSerializeSize(*_tx, PROTOCOL_VERSION); nModSize = _tx->CalculateModifiedSize(nTxSize); nUsageSize = _tx->DynamicMemoryUsage(); hasZerocoins = _tx->ContainsZerocoins(); diff --git a/src/undo.h b/src/undo.h index 1d6c48515063..7003bc525ee7 100644 --- a/src/undo.h +++ b/src/undo.h @@ -63,7 +63,7 @@ class TxInUndoDeserializer TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; -static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_SIZE_CURRENT / ::GetSerializeSize(CTxIn(), SER_NETWORK, PROTOCOL_VERSION); +static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_SIZE_CURRENT / ::GetSerializeSize(CTxIn(), PROTOCOL_VERSION); /** Undo information for a CTransaction */ class CTxUndo diff --git a/src/validation.cpp b/src/validation.cpp index 4aefc393a2c8..15373f854c72 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1679,7 +1679,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd } vPos.emplace_back(tx.GetHash(), pos); - pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); + pos.nTxOffset += ::GetSerializeSize(tx, CLIENT_VERSION); } // Push new tree anchor @@ -1752,7 +1752,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd if (pindex->GetUndoPos().IsNull() || !pindex->IsValid(BLOCK_VALID_SCRIPTS)) { if (pindex->GetUndoPos().IsNull()) { FlatFilePos diskPosBlock; - if (!FindUndoPos(state, pindex->nFile, diskPosBlock, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) + if (!FindUndoPos(state, pindex->nFile, diskPosBlock, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) return error("ConnectBlock() : FindUndoPos failed"); if (!UndoWriteToDisk(blockundo, diskPosBlock, pindex->pprev->GetBlockHash())) return AbortNode(state, "Failed to write undo data"); @@ -2778,7 +2778,7 @@ bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bo // Size limits unsigned int nMaxBlockSize = MAX_BLOCK_SIZE_CURRENT; - const unsigned int nBlockSize = ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION); + const unsigned int nBlockSize = ::GetSerializeSize(block, PROTOCOL_VERSION); if (block.vtx.empty() || block.vtx.size() > nMaxBlockSize || nBlockSize > nMaxBlockSize) return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); @@ -3345,7 +3345,7 @@ static bool AcceptBlock(const CBlock& block, CValidationState& state, CBlockInde // Write block to history file try { - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; if (dbp != nullptr) blockPos = *dbp; @@ -3401,7 +3401,7 @@ bool ProcessNewBlock(CValidationState& state, const std::shared_ptr(Params().GenesisBlock()); // Start new block file - unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); + unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; CValidationState state; if (!FindBlockPos(state, blockPos, nBlockSize + 8, 0, block.GetBlockTime())) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e5f30c04e174..67a321fbdb3b 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -2955,7 +2955,7 @@ bool CWallet::FundTransaction(CMutableTransaction& tx, CAmount& nFeeRet, bool ov coinControl.nFeeRate = specificFeeRate; const int nExtraSize = tx.isSaplingVersion() ? - (int)(GetSerializeSizeNetwork(tx.sapData) + GetSerializeSizeNetwork(tx.extraPayload)) : 0; + (int)(GetSerializeSize(tx.sapData) + GetSerializeSize(tx.extraPayload)) : 0; for (const CTxIn& txin : tx.vin) { coinControl.Select(txin.prevout); @@ -3174,7 +3174,7 @@ bool CWallet::CreateTransaction(const std::vector& vecSend, } // account for additional payloads in fee calculation - const unsigned int nBytes = ::GetSerializeSize(txNew, SER_NETWORK, PROTOCOL_VERSION) + nExtraSize; + const unsigned int nBytes = ::GetSerializeSize(txNew, PROTOCOL_VERSION) + nExtraSize; CAmount nFeeNeeded = std::max(nFeePay, GetMinimumFee(nBytes, nTxConfirmTarget, mempool)); // Remove scriptSigs to eliminate the fee calculation dummy signatures @@ -3234,7 +3234,7 @@ bool CWallet::CreateTransaction(const std::vector& vecSend, } // Limit size - if (::GetSerializeSize(txNew, SER_NETWORK, PROTOCOL_VERSION) >= MAX_STANDARD_TX_SIZE) { + if (::GetSerializeSize(txNew, PROTOCOL_VERSION) >= MAX_STANDARD_TX_SIZE) { strFailReason = _("Transaction too large"); return false; } @@ -3342,7 +3342,7 @@ bool CWallet::CreateCoinStake( txNew.vin.emplace_back(stakeInput.GetTxIn()); // Limit size - unsigned int nBytes = ::GetSerializeSize(txNew, SER_NETWORK, PROTOCOL_VERSION); + unsigned int nBytes = ::GetSerializeSize(txNew, PROTOCOL_VERSION); if (nBytes >= DEFAULT_BLOCK_MAX_SIZE / 5) return error("%s : exceeded coinstake size limit", __func__); From ace7d14ef4a09d49628f27ccdc581ea91c788980 Mon Sep 17 00:00:00 2001 From: Ben Woosley Date: Thu, 14 Jun 2018 19:48:30 -0500 Subject: [PATCH 06/40] Drop minor GetSerializeSize template Now that `GetType()` is not propagated, the benefits are not worth the code. --- src/serialize.h | 12 +++++++----- src/validation.cpp | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 1fb864962d81..73e35c344d4a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1134,15 +1134,17 @@ size_t GetSerializeSize(const T& t, int nVersion = 0) return (CSizeComputer(nVersion) << t).size(); } -template -size_t GetSerializeSize(const S& s, const T& t) +template +size_t GetSerializeSizeMany(int nVersion, const T&... t) { - return (CSizeComputer(s.GetVersion()) << t).size(); + CSizeComputer sc(nVersion); + SerializeMany(sc, t...); + return sc.size(); } /** - * optional - */ +* optional +*/ template unsigned int GetSerializeSize(const Optional &item) { diff --git a/src/validation.cpp b/src/validation.cpp index 15373f854c72..496e3b3010d4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -791,7 +791,7 @@ bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) return error("WriteBlockToDisk : OpenBlockFile failed"); // Write index header - unsigned int nSize = GetSerializeSize(fileout, block); + unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); fileout << Params().MessageStart() << nSize; // Write block @@ -1228,7 +1228,7 @@ bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint25 return error("%s : OpenUndoFile failed", __func__); // Write index header - unsigned int nSize = GetSerializeSize(fileout, blockundo); + unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); fileout << Params().MessageStart() << nSize; // Write undo data From 6bb135e3574de514d386a02e6d675e26e8c49469 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 7 Jul 2017 15:48:13 -0700 Subject: [PATCH 07/40] Introduce new serialization macros without casts This new approach uses a static method which takes the object as a argument. This has the advantage that its constness can be a template parameter, allowing a single implementation that sees the object as const for serialization and non-const for deserialization, without casts. More boilerplate is included in the new macro as well. --- src/serialize.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index 73e35c344d4a..8b3f1db9952e 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -198,6 +198,28 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } SerializationOp(s, CSerActionUnserialize()); \ } +/** + * Implement the Serialize and Unserialize methods by delegating to a single templated + * static method that takes the to-be-(de)serialized object as a parameter. This approach + * has the advantage that the constness of the object becomes a template parameter, and + * thus allows a single implementation that sees the object as const for serializing + * and non-const for deserializing, without casts. + */ +#define SERIALIZE_METHODS(cls, obj) \ + template \ + void Serialize(Stream& s) const \ + { \ + static_assert(std::is_same::value, "Serialize type mismatch"); \ + SerializationOps(*this, s, CSerActionSerialize()); \ + } \ + template \ + void Unserialize(Stream& s) \ + { \ + static_assert(std::is_same::value, "Unserialize type mismatch"); \ + SerializationOps(*this, s, CSerActionUnserialize()); \ + } \ + template \ + static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \ /* * Basic Types From ace389599136ba2f0ac5da933d683d0f734bd951 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 7 Jul 2017 16:08:51 -0700 Subject: [PATCH 08/40] Convert addrdb/addrman to new serialization --- src/addrdb.h | 10 +--------- src/addrman.h | 12 ++++-------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/addrdb.h b/src/addrdb.h index 66284b96bd1f..b2523d2a78e3 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -45,15 +45,7 @@ class CBanEntry nCreateTime = nCreateTimeIn; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nVersion); - READWRITE(nCreateTime); - READWRITE(nBanUntil); - READWRITE(banReason); - } + SERIALIZE_METHODS(CBanEntry, obj) { READWRITE(obj.nVersion, obj.nCreateTime, obj.nBanUntil, obj.banReason); } void SetNull() { diff --git a/src/addrman.h b/src/addrman.h index b78d391959aa..f8b9224eecf8 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -55,15 +55,11 @@ class CAddrInfo : public CAddress friend class CAddrMan; public: - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CAddrInfo, obj) { - READWRITEAS(CAddress, *this); - READWRITE(source); - READWRITE(nLastSuccess); - READWRITE(nAttempts); + READWRITEAS(CAddress, obj); + READWRITE(obj.source, obj.nLastSuccess, obj.nAttempts); } void Init() @@ -310,7 +306,7 @@ class CAddrMan * This format is more complex, but significantly smaller (at most 1.5 MiB), and supports * changes to the ADDRMAN_ parameters without breaking the on-disk structure. * - * We don't use ADD_SERIALIZE_METHODS since the serialization and deserialization code has + * We don't use SERIALIZE_METHODS since the serialization and deserialization code has * very little in common. */ template From 39c58a1e79b0b5a4d2e6135bebc012d08c2d9e82 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 8 Jan 2020 08:56:19 -0800 Subject: [PATCH 09/40] Add a generic approach for (de)serialization of objects using code in other classes This adds the (internal) Wrapper class, and the Using function that uses it. Given a class F that implements Ser(stream, const object&) and Unser(stream, object&) functions, this permits writing e.g. READWRITE(Using(object)). --- src/serialize.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index 8b3f1db9952e..f51a636e049c 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -470,6 +470,32 @@ I ReadVarInt(Stream& is) } } +/** Simple wrapper class to serialize objects using a formatter; used by Using(). */ +template +class Wrapper +{ + static_assert(std::is_lvalue_reference::value, "Wrapper needs an lvalue reference type T"); +protected: + T m_object; +public: + explicit Wrapper(T obj) : m_object(obj) {} + template void Serialize(Stream &s) const { Formatter().Ser(s, m_object); } + template void Unserialize(Stream &s) { Formatter().Unser(s, m_object); } +}; + +/** Cause serialization/deserialization of an object to be done using a specified formatter class. + * + * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for + * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside + * READWRITE, or directly with << and >> operators), can then use Using(object). + * + * This works by constructing a Wrapper-wrapped version of object, where T is + * const during serialization, and non-const during deserialization, which maintains const + * correctness. + */ +template +static inline Wrapper Using(T&& t) { return Wrapper(t); } + #define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj)) #define COMPACTSIZE(obj) CCompactSize(REF(obj)) #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) From bbfc55c0b00e60fcc6815ae3e5f465c4692bacf6 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 8 Jan 2020 09:05:44 -0800 Subject: [PATCH 10/40] Convert VARINT to the formatter/Using approach --- src/serialize.h | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index f51a636e049c..8e47a4b4a76d 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -496,27 +496,22 @@ class Wrapper template static inline Wrapper Using(T&& t) { return Wrapper(t); } -#define VARINT(obj, ...) WrapVarInt<__VA_ARGS__>(REF(obj)) +#define VARINT(obj, ...) Using>(obj) #define COMPACTSIZE(obj) CCompactSize(REF(obj)) #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) -template -class CVarInt +/** Serialization wrapper class for integers in VarInt format. */ +template +struct VarIntFormatter { -protected: - I& n; - -public: - CVarInt(I& nIn) : n(nIn) {} - - template - void Serialize(Stream& s) const { - WriteVarInt(s, n); + template void Ser(Stream &s, I v) + { + WriteVarInt::type>(s, v); } - template - void Unserialize(Stream& s) { - n = ReadVarInt(s); + template void Unser(Stream& s, I& v) + { + v = ReadVarInt::type>(s); } }; @@ -602,9 +597,6 @@ class LimitedString } }; -template -CVarInt WrapVarInt(I& n) { return CVarInt{n}; } - template BigEndian WrapBigEndian(I& n) { return BigEndian(n); } From 7376a95d8f3dbb5791da1692bf0ef5dabc3370a4 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 7 Jul 2017 20:27:39 -0700 Subject: [PATCH 11/40] Convert chain to new serialization (Without converting CDiskBlockIndex) --- src/chain.h | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/chain.h b/src/chain.h index 8a3e8ef6de38..631c9718396e 100644 --- a/src/chain.h +++ b/src/chain.h @@ -41,18 +41,15 @@ class CBlockFileInfo uint64_t nTimeFirst; //!< earliest time of block in file uint64_t nTimeLast; //!< latest time of block in file - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CBlockFileInfo, obj) { - READWRITE(VARINT(nBlocks)); - READWRITE(VARINT(nSize)); - READWRITE(VARINT(nUndoSize)); - READWRITE(VARINT(nHeightFirst)); - READWRITE(VARINT(nHeightLast)); - READWRITE(VARINT(nTimeFirst)); - READWRITE(VARINT(nTimeLast)); + READWRITE(VARINT(obj.nBlocks)); + READWRITE(VARINT(obj.nSize)); + READWRITE(VARINT(obj.nUndoSize)); + READWRITE(VARINT(obj.nHeightFirst)); + READWRITE(VARINT(obj.nHeightLast)); + READWRITE(VARINT(obj.nTimeFirst)); + READWRITE(VARINT(obj.nTimeLast)); } void SetNull() From 3e38199cf36d5d59fde842b480504b05ec9bb7b5 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 8 Jul 2017 16:43:34 -0700 Subject: [PATCH 12/40] Move compressor utility functions out of class --- src/compressor.cpp | 30 +++++++++++++++++++----------- src/compressor.h | 37 ++++++++++++------------------------- src/test/compress_tests.cpp | 8 ++++---- 3 files changed, 35 insertions(+), 40 deletions(-) diff --git a/src/compressor.cpp b/src/compressor.cpp index a54f7ab6757f..18d58649e33b 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -10,7 +10,15 @@ #include "pubkey.h" #include "script/standard.h" -bool CScriptCompressor::IsToKeyID(CKeyID& hash) const +/* + * These check for scripts for which a special case with a shorter encoding is defined. + * They are implemented separately from the CScript test, as these test for exact byte + * sequence correspondences, and are more strict. For example, IsToPubKey also verifies + * whether the public key is valid (as invalid ones cannot be represented in compressed + * form). + */ + +static bool IsToKeyID(const CScript& script, CKeyID &hash) { if (script.size() == 25 && script[0] == OP_DUP && script[1] == OP_HASH160 && script[2] == 20 && script[23] == OP_EQUALVERIFY && script[24] == OP_CHECKSIG) { memcpy(&hash, &script[3], 20); @@ -19,7 +27,7 @@ bool CScriptCompressor::IsToKeyID(CKeyID& hash) const return false; } -bool CScriptCompressor::IsToScriptID(CScriptID& hash) const +static bool IsToScriptID(const CScript& script, CScriptID &hash) { if (script.size() == 23 && script[0] == OP_HASH160 && script[1] == 20 && script[22] == OP_EQUAL) { memcpy(&hash, &script[2], 20); @@ -28,7 +36,7 @@ bool CScriptCompressor::IsToScriptID(CScriptID& hash) const return false; } -bool CScriptCompressor::IsToPubKey(CPubKey& pubkey) const +static bool IsToPubKey(const CScript& script, CPubKey &pubkey) { if (script.size() == 35 && script[0] == 33 && script[34] == OP_CHECKSIG && (script[1] == 0x02 || script[1] == 0x03)) { pubkey.Set(&script[1], &script[34]); @@ -41,24 +49,24 @@ bool CScriptCompressor::IsToPubKey(CPubKey& pubkey) const return false; } -bool CScriptCompressor::Compress(std::vector& out) const +bool CompressScript(const CScript& script, std::vector &out) { CKeyID keyID; - if (IsToKeyID(keyID)) { + if (IsToKeyID(script, keyID)) { out.resize(21); out[0] = 0x00; memcpy(&out[1], &keyID, 20); return true; } CScriptID scriptID; - if (IsToScriptID(scriptID)) { + if (IsToScriptID(script, scriptID)) { out.resize(21); out[0] = 0x01; memcpy(&out[1], &scriptID, 20); return true; } CPubKey pubkey; - if (IsToPubKey(pubkey)) { + if (IsToPubKey(script, pubkey)) { out.resize(33); memcpy(&out[1], &pubkey[1], 32); if (pubkey[0] == 0x02 || pubkey[0] == 0x03) { @@ -72,7 +80,7 @@ bool CScriptCompressor::Compress(std::vector& out) const return false; } -unsigned int CScriptCompressor::GetSpecialSize(unsigned int nSize) const +unsigned int GetSpecialScriptSize(unsigned int nSize) { if (nSize == 0 || nSize == 1) return 20; @@ -81,7 +89,7 @@ unsigned int CScriptCompressor::GetSpecialSize(unsigned int nSize) const return 0; } -bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector& in) +bool DecompressScript(CScript& script, unsigned int nSize, const std::vector &in) { switch (nSize) { case 0x00: @@ -135,7 +143,7 @@ bool CScriptCompressor::Decompress(unsigned int nSize, const std::vector &out); +unsigned int GetSpecialScriptSize(unsigned int nSize); +bool DecompressScript(CScript& script, unsigned int nSize, const std::vector &out); + +uint64_t CompressAmount(uint64_t nAmount); +uint64_t DecompressAmount(uint64_t nAmount); + /** Compact serializer for scripts. * * It detects common cases and encodes them much more efficiently. @@ -38,24 +45,7 @@ class CScriptCompressor */ static const unsigned int nSpecialScripts = 6; - CScript& script; - -protected: - /** - * These check for scripts for which a special case with a shorter encoding is defined. - * They are implemented separately from the CScript test, as these test for exact byte - * sequence correspondences, and are more strict. For example, IsToPubKey also verifies - * whether the public key is valid (as invalid ones cannot be represented in compressed - * form). - */ - bool IsToKeyID(CKeyID& hash) const; - bool IsToScriptID(CScriptID& hash) const; - bool IsToPubKey(CPubKey& pubkey) const; - - bool Compress(std::vector& out) const; - unsigned int GetSpecialSize(unsigned int nSize) const; - bool Decompress(unsigned int nSize, const std::vector& out); - + CScript &script; public: CScriptCompressor(CScript& scriptIn) : script(scriptIn) {} @@ -63,7 +53,7 @@ class CScriptCompressor void Serialize(Stream& s) const { std::vector compr; - if (Compress(compr)) { + if (CompressScript(script, compr)) { s << MakeSpan(compr); return; } @@ -78,9 +68,9 @@ class CScriptCompressor unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { - std::vector vch(GetSpecialSize(nSize), 0x00); + std::vector vch(GetSpecialScriptSize(nSize), 0x00); s >> MakeSpan(vch); - Decompress(nSize, vch); + DecompressScript(script, nSize, vch); return; } nSize -= nSpecialScripts; @@ -102,10 +92,7 @@ class CTxOutCompressor CTxOut& txout; public: - static uint64_t CompressAmount(uint64_t nAmount); - static uint64_t DecompressAmount(uint64_t nAmount); - - CTxOutCompressor(CTxOut& txoutIn) : txout(txoutIn) {} + explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { } ADD_SERIALIZE_METHODS; diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index c0bcbb7ba67b..14dd6343b9af 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -25,16 +25,16 @@ BOOST_FIXTURE_TEST_SUITE(compress_tests, BasicTestingSetup) bool static TestEncode(uint64_t in) { - return in == CTxOutCompressor::DecompressAmount(CTxOutCompressor::CompressAmount(in)); + return in == DecompressAmount(CompressAmount(in)); } bool static TestDecode(uint64_t in) { - return in == CTxOutCompressor::CompressAmount(CTxOutCompressor::DecompressAmount(in)); + return in == CompressAmount(DecompressAmount(in)); } bool static TestPair(uint64_t dec, uint64_t enc) { - return CTxOutCompressor::CompressAmount(dec) == enc && - CTxOutCompressor::DecompressAmount(enc) == dec; + return CompressAmount(dec) == enc && + DecompressAmount(enc) == dec; } BOOST_AUTO_TEST_CASE(compress_amounts) From aa3599190636c27f2f71fa65027f4dc35233f0de Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 18 Jan 2020 07:32:31 -0800 Subject: [PATCH 13/40] Add FORMATTER_METHODS, similar to SERIALIZE_METHODS, but for formatters --- src/serialize.h | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 8e47a4b4a76d..65570c5cbfc2 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -198,6 +198,30 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } SerializationOp(s, CSerActionUnserialize()); \ } +/** + * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). + * + * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic + * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when + * deserializing). + * + * Example use: + * struct FooFormatter { + * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); } + * } + * would define a class FooFormatter that defines a serialization of Class objects consisting + * of serializing its val1 member using the default serialization, and its val2 member using + * VARINT serialization. That FooFormatter can then be used in statements like + * READWRITE(Using(obj.bla)). + */ +#define FORMATTER_METHODS(cls, obj) \ + template \ + static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \ + template \ + static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \ + template \ + static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \ + /** * Implement the Serialize and Unserialize methods by delegating to a single templated * static method that takes the to-be-(de)serialized object as a parameter. This approach @@ -210,16 +234,15 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } void Serialize(Stream& s) const \ { \ static_assert(std::is_same::value, "Serialize type mismatch"); \ - SerializationOps(*this, s, CSerActionSerialize()); \ + Ser(s, *this); \ } \ template \ void Unserialize(Stream& s) \ { \ static_assert(std::is_same::value, "Unserialize type mismatch"); \ - SerializationOps(*this, s, CSerActionUnserialize()); \ + Unser(s, *this); \ } \ - template \ - static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \ + FORMATTER_METHODS(cls, obj) /* * Basic Types From c2fdeaf54f9960c3c6fc24a3876eab259d836529 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 18 Jan 2020 07:32:58 -0800 Subject: [PATCH 14/40] Convert compression.h to new serialization framework --- src/coins.h | 4 +-- src/compressor.h | 58 ++++++++++++----------------------- src/test/fuzz/deserialize.cpp | 2 +- src/txdb.cpp | 2 +- src/undo.h | 4 +-- 5 files changed, 26 insertions(+), 44 deletions(-) diff --git a/src/coins.h b/src/coins.h index e86e8415eba3..315a7c14f96e 100644 --- a/src/coins.h +++ b/src/coins.h @@ -69,7 +69,7 @@ class Coin assert(!IsSpent()); uint32_t code = nHeight * 4 + (fCoinBase ? 2 : 0) + (fCoinStake ? 1 : 0); ::Serialize(s, VARINT(code)); - ::Serialize(s, CTxOutCompressor(REF(out))); + ::Serialize(s, Using(out)); } template @@ -79,7 +79,7 @@ class Coin nHeight = code >> 2; fCoinBase = code & 2; fCoinStake = code & 1; - ::Unserialize(s, CTxOutCompressor(out)); + ::Unserialize(s, Using(out)); } bool IsSpent() const { diff --git a/src/compressor.h b/src/compressor.h index d7b5f3279f5d..d85925d2bf1f 100644 --- a/src/compressor.h +++ b/src/compressor.h @@ -12,10 +12,6 @@ #include "serialize.h" #include "span.h" -class CKeyID; -class CPubKey; -class CScriptID; - bool CompressScript(const CScript& script, std::vector &out); unsigned int GetSpecialScriptSize(unsigned int nSize); bool DecompressScript(CScript& script, unsigned int nSize, const std::vector &out); @@ -34,9 +30,8 @@ uint64_t DecompressAmount(uint64_t nAmount); * Other scripts up to 121 bytes require 1 byte + script length. Above * that, scripts up to 16505 bytes require 2 bytes + script length. */ -class CScriptCompressor +struct ScriptCompression { -private: /** * make this static for now (there are only 6 special scripts defined) * this can potentially be extended together with a new nVersion for @@ -45,13 +40,8 @@ class CScriptCompressor */ static const unsigned int nSpecialScripts = 6; - CScript &script; -public: - CScriptCompressor(CScript& scriptIn) : script(scriptIn) {} - - template - void Serialize(Stream& s) const - { + template + void Ser(Stream &s, const CScript& script) { std::vector compr; if (CompressScript(script, compr)) { s << MakeSpan(compr); @@ -62,9 +52,8 @@ class CScriptCompressor s << MakeSpan(script); } - template - void Unserialize(Stream& s) - { + template + void Unser(Stream &s, CScript& script) { unsigned int nSize = 0; s >> VARINT(nSize); if (nSize < nSpecialScripts) { @@ -85,31 +74,24 @@ class CScriptCompressor } }; -/** wrapper for CTxOut that provides a more compact serialization */ -class CTxOutCompressor +struct AmountCompression { -private: - CTxOut& txout; - -public: - explicit CTxOutCompressor(CTxOut &txoutIn) : txout(txoutIn) { } - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + template void Ser(Stream& s, I val) { - if (!ser_action.ForRead()) { - uint64_t nVal = CompressAmount(txout.nValue); - READWRITE(VARINT(nVal)); - } else { - uint64_t nVal = 0; - READWRITE(VARINT(nVal)); - txout.nValue = DecompressAmount(nVal); - } - CScriptCompressor cscript(REF(txout.scriptPubKey)); - READWRITE(cscript); + s << VARINT(CompressAmount(val)); + } + template void Unser(Stream& s, I& val) + { + uint64_t v; + s >> VARINT(v); + val = DecompressAmount(v); } }; +/** wrapper for CTxOut that provides a more compact serialization */ +struct TxOutCompression +{ + FORMATTER_METHODS(CTxOut, obj) { READWRITE(Using(obj.nValue), Using(obj.scriptPubKey)); } +}; + #endif // BITCOIN_COMPRESSOR_H diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 8f8b85fa1b42..578b23be9f78 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -175,7 +175,7 @@ void test_one_input(std::vector buffer) DeserializeFromFuzzingInput(buffer, dbi); #elif TXOUTCOMPRESSOR_DESERIALIZE CTxOut to; - CTxOutCompressor toc(to); + auto toc = Using(to); DeserializeFromFuzzingInput(buffer, toc); #else #error Need at least one fuzz target to compile diff --git a/src/txdb.cpp b/src/txdb.cpp index 2035c1c75081..486f42e83168 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -487,7 +487,7 @@ class CCoins vout.assign(vAvail.size(), CTxOut()); for (unsigned int i = 0; i < vAvail.size(); i++) { if (vAvail[i]) - ::Unserialize(s, CTxOutCompressor(vout[i])); + ::Unserialize(s, Using(vout[i])); } // coinbase height ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); diff --git a/src/undo.h b/src/undo.h index 7003bc525ee7..326e1c8461f5 100644 --- a/src/undo.h +++ b/src/undo.h @@ -31,7 +31,7 @@ class TxInUndoSerializer // Required to maintain compatibility with older undo format. ::Serialize(s, (unsigned char)0); } - ::Serialize(s, CTxOutCompressor(REF(txout->out))); + ::Serialize(s, Using(REF(txout->out))); } TxInUndoSerializer(const Coin* coin) : txout(coin) {} @@ -57,7 +57,7 @@ class TxInUndoDeserializer unsigned int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } - ::Unserialize(s, CTxOutCompressor(REF(txout->out))); + ::Unserialize(s, Using(REF(txout->out))); } TxInUndoDeserializer(Coin* coin) : txout(coin) {} From df4e1bab96945e7c4f2717ab7f7ea785028f1948 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 6 Sep 2017 11:25:03 -0700 Subject: [PATCH 15/40] Add a constant for the maximum vector allocation (5 Mbyte) --- src/serialize.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 65570c5cbfc2..fb491ee7b8e4 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -34,6 +34,9 @@ class CScript; static const unsigned int MAX_SIZE = 0x02000000; +/** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */ +static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; + /** * Dummy data type to identify deserializing constructors. * @@ -890,7 +893,7 @@ void Unserialize_impl(Stream& is, prevector& v, const V&) unsigned int nMid = 0; while (nMid < nSize) { - nMid += 5000000 / sizeof(T); + nMid += MAX_VECTOR_ALLOCATE / sizeof(T); if (nMid > nSize) nMid = nSize; v.resize_uninitialized(nMid); @@ -956,7 +959,7 @@ void Unserialize_impl(Stream& is, std::vector& v, const V&) unsigned int i = 0; unsigned int nMid = 0; while (nMid < nSize) { - nMid += 5000000 / sizeof(T); + nMid += MAX_VECTOR_ALLOCATE / sizeof(T); if (nMid > nSize) nMid = nSize; v.resize(nMid); From 1dfddcefccefe87f3a94e304a59fb50d9efc07eb Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 18 Jan 2020 06:49:14 -0800 Subject: [PATCH 16/40] Add custom vector-element formatter This allows a very compact notation for serialization of vectors whose elements are not serialized using their default encoding. --- src/serialize.h | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index fb491ee7b8e4..02a73fbb414c 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -626,6 +626,53 @@ class LimitedString template BigEndian WrapBigEndian(I& n) { return BigEndian(n); } +/** Formatter to serialize/deserialize vector elements using another formatter + * + * Example: + * struct X { + * std::vector v; + * SERIALIZE_METHODS(X, obj) { READWRITE(Using>(obj.v)); } + * }; + * will define a struct that contains a vector of uint64_t, which is serialized + * as a vector of VarInt-encoded integers. + * + * V is not required to be an std::vector type. It works for any class that + * exposes a value_type, size, reserve, push_back, and const iterators. + */ +template +struct VectorFormatter +{ + template + void Ser(Stream& s, const V& v) + { + WriteCompactSize(s, v.size()); + for (const typename V::value_type& elem : v) { + s << Using(elem); + } + } + + template + void Unser(Stream& s, V& v) + { + v.clear(); + size_t size = ReadCompactSize(s); + size_t allocated = 0; + while (allocated < size) { + // For DoS prevention, do not blindly allocate as much as the stream claims to contain. + // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide + // X MiB of data to make us allocate X+5 Mib. + static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large"); + allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type)); + v.reserve(allocated); + while (v.size() < allocated) { + typename V::value_type val; + s >> Using(val); + v.push_back(std::move(val)); + } + } + }; +}; + /** * Forward declarations */ From a926ba352466a9a8e5d5152ad8060c26d4394d92 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 29 Jan 2020 10:44:52 -0800 Subject: [PATCH 17/40] Make std::vector and prevector reuse the VectorFormatter logic --- src/serialize.h | 49 ++++++++++++++++++------------------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 02a73fbb414c..c31fb851c39f 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -830,6 +830,20 @@ inline void Unserialize(Stream& is, T&& a) a.Unserialize(is); } +/** Default formatter. Serializes objects as themselves. + * + * The vector/prevector serialization code passes this to VectorFormatter + * to enable reusing that logic. It shouldn't be needed elsewhere. + */ +struct DefaultFormatter +{ + template + static void Ser(Stream& s, const T& t) { Serialize(s, t); } + + template + static void Unser(Stream& s, T& t) { Unserialize(s, t); } +}; + /** * string @@ -903,9 +917,7 @@ void Serialize_impl(Stream& os, const prevector& v, const unsigned char&) template void Serialize_impl(Stream& os, const prevector& v, const V&) { - WriteCompactSize(os, v.size()); - for (typename prevector::const_iterator vi = v.begin(); vi != v.end(); ++vi) - ::Serialize(os, (*vi)); + Serialize(os, Using>(v)); } template @@ -934,19 +946,7 @@ void Unserialize_impl(Stream& is, prevector& v, const unsigned char&) template void Unserialize_impl(Stream& is, prevector& v, const V&) { - v.clear(); - unsigned int nSize = ReadCompactSize(is); - unsigned int i = 0; - unsigned int nMid = 0; - while (nMid < nSize) - { - nMid += MAX_VECTOR_ALLOCATE / sizeof(T); - if (nMid > nSize) - nMid = nSize; - v.resize_uninitialized(nMid); - for (; i < nMid; ++i) - Unserialize(is, v[i]); - } + Unserialize(is, Using>(v)); } template @@ -971,9 +971,7 @@ void Serialize_impl(Stream& os, const std::vector& v, const unsigned char& template void Serialize_impl(Stream& os, const std::vector& v, const V&) { - WriteCompactSize(os, v.size()); - for (typename std::vector::const_iterator vi = v.begin(); vi != v.end(); ++vi) - ::Serialize(os, (*vi)); + Serialize(os, Using>(v)); } template @@ -1001,18 +999,7 @@ void Unserialize_impl(Stream& is, std::vector& v, const unsigned char&) template void Unserialize_impl(Stream& is, std::vector& v, const V&) { - v.clear(); - unsigned int nSize = ReadCompactSize(is); - unsigned int i = 0; - unsigned int nMid = 0; - while (nMid < nSize) { - nMid += MAX_VECTOR_ALLOCATE / sizeof(T); - if (nMid > nSize) - nMid = nSize; - v.resize(nMid); - for (; i < nMid; i++) - Unserialize(is, v[i]); - } + Unserialize(is, Using>(v)); } template From e107a0c36bfc134e3a939eeeec54d49340101011 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 29 Jan 2020 11:21:36 -0800 Subject: [PATCH 18/40] Convert undo.h to new serialization framework --- src/undo.h | 79 ++++++++++++------------------------------------------ 1 file changed, 17 insertions(+), 62 deletions(-) diff --git a/src/undo.h b/src/undo.h index 326e1c8461f5..f6779dab6dfd 100644 --- a/src/undo.h +++ b/src/undo.h @@ -13,58 +13,43 @@ #include "primitives/transaction.h" #include "serialize.h" -/** Undo information for a CTxIn +/** Formatter for undo information for a CTxIn + * * Contains the prevout's CTxOut being spent, and its metadata as well * (coinbase/coinstake or not, height). The serialization contains a * dummy value of zero. This is be compatible with older versions which * expect to see the transaction version there. */ -class TxInUndoSerializer +struct TxInUndoFormatter { - const Coin* txout; - -public: - template - void Serialize(Stream& s) const { - ::Serialize(s, VARINT(txout->nHeight * 4 + (txout->fCoinBase ? 2u : 0u) + (txout->fCoinStake ? 1u : 0u))); - if (txout->nHeight > 0) { + template + void Ser(Stream &s, const Coin& txout) { + ::Serialize(s, VARINT(txout.nHeight * 4 + (txout.fCoinBase ? 2u : 0u) + (txout.fCoinStake ? 1u : 0u))); + if (txout.nHeight > 0) { // Required to maintain compatibility with older undo format. ::Serialize(s, (unsigned char)0); } - ::Serialize(s, Using(REF(txout->out))); + ::Serialize(s, Using(txout.out)); } - TxInUndoSerializer(const Coin* coin) : txout(coin) {} -}; - -class TxInUndoDeserializer -{ - Coin* txout; - -public: - template - void Unserialize(Stream& s) - { + template + void Unser(Stream &s, Coin& txout) { unsigned int nCode = 0; ::Unserialize(s, VARINT(nCode)); - txout->nHeight = nCode >> 2; - txout->fCoinBase = nCode & 2; - txout->fCoinStake = nCode & 1; - if (txout->nHeight > 0) { + txout.nHeight = nCode >> 2; + txout.fCoinBase = nCode & 2; + txout.fCoinStake = nCode & 1; + if (txout.nHeight > 0) { // Old versions stored the version number for the last spend of // a transaction's outputs. Non-final spends were indicated with // height = 0. unsigned int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } - ::Unserialize(s, Using(REF(txout->out))); + ::Unserialize(s, Using(txout.out)); } - - TxInUndoDeserializer(Coin* coin) : txout(coin) {} }; -static const size_t MAX_INPUTS_PER_BLOCK = MAX_BLOCK_SIZE_CURRENT / ::GetSerializeSize(CTxIn(), PROTOCOL_VERSION); - /** Undo information for a CTransaction */ class CTxUndo { @@ -72,31 +57,7 @@ class CTxUndo // undo information for all txins std::vector vprevout; - template - void Serialize(Stream& s) const - { - // TODO: avoid reimplementing vector serializer - uint64_t count = vprevout.size(); - ::Serialize(s, COMPACTSIZE(REF(count))); - for (const auto& prevout : vprevout) { - ::Serialize(s, TxInUndoSerializer(&prevout)); - } - } - - template - void Unserialize(Stream& s) - { - // TODO: avoid reimplementing vector deserializer - uint64_t count = 0; - ::Unserialize(s, COMPACTSIZE(count)); - if (count > MAX_INPUTS_PER_BLOCK) { - throw std::ios_base::failure("Too many input undo records"); - } - vprevout.resize(count); - for (auto& prevout : vprevout) { - ::Unserialize(s, TxInUndoDeserializer(&prevout)); - } - } + SERIALIZE_METHODS(CTxUndo, obj) { READWRITE(Using>(obj.vprevout)); } }; /** Undo information for a CBlock */ @@ -105,13 +66,7 @@ class CBlockUndo public: std::vector vtxundo; // for all but the coinbase - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(vtxundo); - } + SERIALIZE_METHODS(CBlockUndo, obj) { READWRITE(obj.vtxundo); } }; #endif // BITCOIN_UNDO_H From bb9903055de6c3da9092c3c1fafe56ff9935953a Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Thu, 6 Feb 2020 19:57:32 -0800 Subject: [PATCH 19/40] Get rid of VARINT default argument This removes the need for the GNU C++ extension of variadic macros. --- src/chain.h | 6 +++--- src/flatfile.h | 2 +- src/rpc/blockchain.cpp | 2 +- src/serialize.h | 5 +++-- src/test/serialize_tests.cpp | 26 +++++++++++++------------- src/txdb.cpp | 2 +- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/chain.h b/src/chain.h index 631c9718396e..97cfec01a01d 100644 --- a/src/chain.h +++ b/src/chain.h @@ -283,13 +283,13 @@ class CDiskBlockIndex : public CBlockIndex { int nSerVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) - READWRITE(VARINT(nSerVersion, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(nSerVersion, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(VARINT(nStatus)); READWRITE(VARINT(nTx)); if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(nFile, VarIntMode::NONNEGATIVE_SIGNED)); if (nStatus & BLOCK_HAVE_DATA) READWRITE(VARINT(nDataPos)); if (nStatus & BLOCK_HAVE_UNDO) diff --git a/src/flatfile.h b/src/flatfile.h index 17eaaeb7b09d..2d1e7e41d504 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -20,7 +20,7 @@ struct FlatFilePos template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(nFile, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(VARINT(nPos)); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index bff9a2f6a709..8c17f679ecbe 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -710,7 +710,7 @@ static void ApplyStats(CCoinsStats &stats, CHashWriter& ss, const uint256& hash, for (const auto& output : outputs) { ss << VARINT(output.first + 1); ss << output.second.out.scriptPubKey; - ss << VARINT(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(output.second.out.nValue, VarIntMode::NONNEGATIVE_SIGNED); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.out.nValue; } diff --git a/src/serialize.h b/src/serialize.h index c31fb851c39f..0c16d81a5aa6 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -522,12 +522,13 @@ class Wrapper template static inline Wrapper Using(T&& t) { return Wrapper(t); } -#define VARINT(obj, ...) Using>(obj) +#define VARINT_MODE(obj, mode) Using>(obj) +#define VARINT(obj) Using>(obj) #define COMPACTSIZE(obj) CCompactSize(REF(obj)) #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) /** Serialization wrapper class for integers in VarInt format. */ -template +template struct VarIntFormatter { template void Ser(Stream &s, I v) diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index c17d51624077..dc6a3be81cca 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -207,8 +207,8 @@ BOOST_AUTO_TEST_CASE(varints) CDataStream ss(SER_DISK, 0); CDataStream::size_type size = 0; for (int i = 0; i < 100000; i++) { - ss << VARINT(i, VarIntMode::NONNEGATIVE_SIGNED); - size += ::GetSerializeSize(VARINT(i, VarIntMode::NONNEGATIVE_SIGNED), 0); + ss << VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED); + size += ::GetSerializeSize(VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED), 0); BOOST_CHECK(size == ss.size()); } @@ -221,7 +221,7 @@ BOOST_AUTO_TEST_CASE(varints) // decode for (int i = 0; i < 100000; i++) { int j = -1; - ss >> VARINT(j, VarIntMode::NONNEGATIVE_SIGNED); + ss >> VARINT_MODE(j, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); } @@ -235,21 +235,21 @@ BOOST_AUTO_TEST_CASE(varints) BOOST_AUTO_TEST_CASE(varints_bitpatterns) { CDataStream ss(SER_DISK, 0); - ss << VARINT(0, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "00"); ss.clear(); - ss << VARINT(0x7f, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); - ss << VARINT((int8_t)0x7f, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); - ss << VARINT(0x80, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "8000"); ss.clear(); + ss << VARINT_MODE(0, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "00"); ss.clear(); + ss << VARINT_MODE(0x7f, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); + ss << VARINT_MODE((int8_t)0x7f, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); + ss << VARINT_MODE(0x80, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "8000"); ss.clear(); ss << VARINT((uint8_t)0x80); BOOST_CHECK_EQUAL(HexStr(ss), "8000"); ss.clear(); - ss << VARINT(0x1234, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "a334"); ss.clear(); - ss << VARINT((int16_t)0x1234, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "a334"); ss.clear(); - ss << VARINT(0xffff, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "82fe7f"); ss.clear(); + ss << VARINT_MODE(0x1234, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "a334"); ss.clear(); + ss << VARINT_MODE((int16_t)0x1234, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "a334"); ss.clear(); + ss << VARINT_MODE(0xffff, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "82fe7f"); ss.clear(); ss << VARINT((uint16_t)0xffff); BOOST_CHECK_EQUAL(HexStr(ss), "82fe7f"); ss.clear(); - ss << VARINT(0x123456, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "c7e756"); ss.clear(); - ss << VARINT((int32_t)0x123456, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "c7e756"); ss.clear(); + ss << VARINT_MODE(0x123456, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "c7e756"); ss.clear(); + ss << VARINT_MODE((int32_t)0x123456, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "c7e756"); ss.clear(); ss << VARINT(0x80123456U); BOOST_CHECK_EQUAL(HexStr(ss), "86ffc7e756"); ss.clear(); ss << VARINT((uint32_t)0x80123456U); BOOST_CHECK_EQUAL(HexStr(ss), "86ffc7e756"); ss.clear(); ss << VARINT(0xffffffff); BOOST_CHECK_EQUAL(HexStr(ss), "8efefefe7f"); ss.clear(); - ss << VARINT(0x7fffffffffffffffLL, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "fefefefefefefefe7f"); ss.clear(); + ss << VARINT_MODE(0x7fffffffffffffffLL, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "fefefefefefefefe7f"); ss.clear(); ss << VARINT(0xffffffffffffffffULL); BOOST_CHECK_EQUAL(HexStr(ss), "80fefefefefefefefe7f"); ss.clear(); } diff --git a/src/txdb.cpp b/src/txdb.cpp index 486f42e83168..469405470bb9 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -490,7 +490,7 @@ class CCoins ::Unserialize(s, Using(vout[i])); } // coinbase height - ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); + ::Unserialize(s, VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); } }; From 4e2afad1a65719cae33820e6611144eed3c7debc Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Mon, 3 Feb 2020 19:49:10 -0800 Subject: [PATCH 20/40] Convert CCompactSize to proper formatter --- src/serialize.h | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 0c16d81a5aa6..62e7d9b1170c 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -524,7 +524,7 @@ static inline Wrapper Using(T&& t) { return Wrapper>(obj) #define VARINT(obj) Using>(obj) -#define COMPACTSIZE(obj) CCompactSize(REF(obj)) +#define COMPACTSIZE(obj) Using(obj) #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) /** Serialization wrapper class for integers in VarInt format. */ @@ -576,21 +576,26 @@ class BigEndian } }; -class CCompactSize +/** Formatter for integers in CompactSize format. */ +struct CompactSizeFormatter { -protected: - uint64_t &n; -public: - CCompactSize(uint64_t& nIn) : n(nIn) { } - - template - void Serialize(Stream &s) const { - WriteCompactSize(s, n); + template + void Unser(Stream& s, I& v) + { + uint64_t n = ReadCompactSize(s); + if (n < std::numeric_limits::min() || n > std::numeric_limits::max()) { + throw std::ios_base::failure("CompactSize exceeds limit of type"); + } + v = n; } - template - void Unserialize(Stream& s) { - n = ReadCompactSize(s); + template + void Ser(Stream& s, I v) + { + static_assert(std::is_unsigned::value, "CompactSize only supported for unsigned integers"); + static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), "CompactSize only supports 64-bit integers and below"); + + WriteCompactSize(s, v); } }; From fd29a508c60a96dbede0571a98a9ae6b87eba570 Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Sat, 15 Feb 2020 19:09:09 -0800 Subject: [PATCH 21/40] Make VectorFormatter support stateful formatters --- src/prevector.h | 9 +++++++-- src/serialize.h | 11 ++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/prevector.h b/src/prevector.h index 386a1781a27d..190e63dcf900 100644 --- a/src/prevector.h +++ b/src/prevector.h @@ -422,15 +422,20 @@ class prevector { return first; } - void push_back(const T& value) { + template + void emplace_back(Args&&... args) { size_type new_size = size() + 1; if (capacity() < new_size) { change_capacity(new_size + (new_size >> 1)); } - new(item_ptr(size())) T(value); + new(item_ptr(size())) T(std::forward(args)...); _size++; } + void push_back(const T& value) { + emplace_back(value); + } + void pop_back() { erase(end() - 1, end()); } diff --git a/src/serialize.h b/src/serialize.h index 62e7d9b1170c..dbacdede3013 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -643,7 +643,7 @@ BigEndian WrapBigEndian(I& n) { return BigEndian(n); } * as a vector of VarInt-encoded integers. * * V is not required to be an std::vector type. It works for any class that - * exposes a value_type, size, reserve, push_back, and const iterators. + * exposes a value_type, size, reserve, emplace_back, back, and const iterators. */ template struct VectorFormatter @@ -651,15 +651,17 @@ struct VectorFormatter template void Ser(Stream& s, const V& v) { + Formatter formatter; WriteCompactSize(s, v.size()); for (const typename V::value_type& elem : v) { - s << Using(elem); + formatter.Ser(s, elem); } } template void Unser(Stream& s, V& v) { + Formatter formatter; v.clear(); size_t size = ReadCompactSize(s); size_t allocated = 0; @@ -671,9 +673,8 @@ struct VectorFormatter allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type)); v.reserve(allocated); while (v.size() < allocated) { - typename V::value_type val; - s >> Using(val); - v.push_back(std::move(val)); + v.emplace_back(); + formatter.Unser(s, v.back()); } } }; From d6380c44beb92caac53fe82029d5f18a0287aead Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sat, 15 Feb 2020 19:48:42 -0800 Subject: [PATCH 22/40] Add CustomUintFormatter --- src/serialize.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index dbacdede3013..59d1fd0b68aa 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -542,6 +542,28 @@ struct VarIntFormatter } }; +template +struct CustomUintFormatter +{ + static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); + static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes)); + + template void Ser(Stream& s, I v) + { + if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); + uint64_t raw = htole64(v); + s.write((const char*)&raw, Bytes); + } + + template void Unser(Stream& s, I& v) + { + static_assert(std::numeric_limits::max() >= MAX && std::numeric_limits::min() <= 0, "CustomUintFormatter type too small"); + uint64_t raw = 0; + s.read((char*)&raw, Bytes); + v = le64toh(raw); + } +}; + /** Serialization wrapper class for big-endian integers. * * Use this wrapper around integer types that are stored in memory in native From 806213a7f85ab88bad63e76a43538869b9fcfc22 Mon Sep 17 00:00:00 2001 From: Samer Afach Date: Mon, 17 Feb 2020 20:53:50 +0100 Subject: [PATCH 23/40] Fix a violation of C++ standard rules that unions cannot be switched. --- src/serialize.h | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 59d1fd0b68aa..19af1bb049b8 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -137,27 +138,27 @@ template inline uint64_t ser_readdata64(Stream &s) } inline uint64_t ser_double_to_uint64(double x) { - union { double x; uint64_t y; } tmp; - tmp.x = x; - return tmp.y; + uint64_t tmp; + std::memcpy(&tmp, &x, sizeof(x)); + return tmp; } inline uint32_t ser_float_to_uint32(float x) { - union { float x; uint32_t y; } tmp; - tmp.x = x; - return tmp.y; + uint32_t tmp; + std::memcpy(&tmp, &x, sizeof(x)); + return tmp; } inline double ser_uint64_to_double(uint64_t y) { - union { double x; uint64_t y; } tmp; - tmp.y = y; - return tmp.x; + double tmp; + std::memcpy(&tmp, &y, sizeof(y)); + return tmp; } inline float ser_uint32_to_float(uint32_t y) { - union { float x; uint32_t y; } tmp; - tmp.y = y; - return tmp.x; + float tmp; + std::memcpy(&tmp, &y, sizeof(y)); + return tmp; } From 3765d6c7b778bb9f0e5eb4f825cac31d1ca1aa7e Mon Sep 17 00:00:00 2001 From: Samer Afach Date: Wed, 19 Feb 2020 18:44:46 +0100 Subject: [PATCH 24/40] Add static_asserts to ser_X_to_Y() methods --- src/serialize.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index 19af1bb049b8..bab361a1b353 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -140,24 +140,28 @@ inline uint64_t ser_double_to_uint64(double x) { uint64_t tmp; std::memcpy(&tmp, &x, sizeof(x)); + static_assert(sizeof(tmp) == sizeof(x), "double and uint64_t assumed to have the same size"); return tmp; } inline uint32_t ser_float_to_uint32(float x) { uint32_t tmp; std::memcpy(&tmp, &x, sizeof(x)); + static_assert(sizeof(tmp) == sizeof(x), "float and uint32_t assumed to have the same size"); return tmp; } inline double ser_uint64_to_double(uint64_t y) { double tmp; std::memcpy(&tmp, &y, sizeof(y)); + static_assert(sizeof(tmp) == sizeof(y), "double and uint64_t assumed to have the same size"); return tmp; } inline float ser_uint32_to_float(uint32_t y) { float tmp; std::memcpy(&tmp, &y, sizeof(y)); + static_assert(sizeof(tmp) == sizeof(y), "float and uint32_t assumed to have the same size"); return tmp; } From c4d62280ec84d7f34bec4bc76b47e83a38349e03 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 11 Mar 2020 08:30:08 -0700 Subject: [PATCH 25/40] Merge BigEndian functionality into CustomUintFormatter --- src/netaddress.h | 5 ++--- src/serialize.h | 57 ++++++++++++++---------------------------------- 2 files changed, 18 insertions(+), 44 deletions(-) diff --git a/src/netaddress.h b/src/netaddress.h index c6133eb90394..ea3a2d99ff68 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -169,10 +169,9 @@ class CService : public CNetAddr ADD_SERIALIZE_METHODS; template - inline void SerializationOp(Stream& s, Operation ser_action) - { + inline void SerializationOp(Stream& s, Operation ser_action) { READWRITE(ip); - READWRITE(WrapBigEndian(port)); + READWRITE(Using>(port)); } }; diff --git a/src/serialize.h b/src/serialize.h index bab361a1b353..c451b6e30f8e 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -547,7 +547,7 @@ struct VarIntFormatter } }; -template +template struct CustomUintFormatter { static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); @@ -556,52 +556,30 @@ struct CustomUintFormatter template void Ser(Stream& s, I v) { if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); - uint64_t raw = htole64(v); - s.write((const char*)&raw, Bytes); + if (BigEndian) { + uint64_t raw = htobe64(v); + s.write(((const char*)&raw) + 8 - Bytes, Bytes); + } else { + uint64_t raw = htole64(v); + s.write((const char*)&raw, Bytes); + } } template void Unser(Stream& s, I& v) { static_assert(std::numeric_limits::max() >= MAX && std::numeric_limits::min() <= 0, "CustomUintFormatter type too small"); uint64_t raw = 0; - s.read((char*)&raw, Bytes); - v = le64toh(raw); + if (BigEndian) { + s.read(((char*)&raw) + 8 - Bytes, Bytes); + v = be64toh(raw); + } else { + s.read((char*)&raw, Bytes); + v = le64toh(raw); + } } }; -/** Serialization wrapper class for big-endian integers. - * - * Use this wrapper around integer types that are stored in memory in native - * byte order, but serialized in big endian notation. This is only intended - * to implement serializers that are compatible with existing formats, and - * its use is not recommended for new data structures. - * - * Only 16-bit types are supported for now. - */ -template -class BigEndian -{ -protected: - I& m_val; -public: - explicit BigEndian(I& val) : m_val(val) - { - static_assert(std::is_unsigned::value, "BigEndian type must be unsigned integer"); - static_assert(sizeof(I) == 2 && std::numeric_limits::min() == 0 && std::numeric_limits::max() == std::numeric_limits::max(), "Unsupported BigEndian size"); - } - - template - void Serialize(Stream& s) const - { - ser_writedata16be(s, m_val); - } - - template - void Unserialize(Stream& s) - { - m_val = ser_readdata16be(s); - } -}; +template using BigEndianFormatter = CustomUintFormatter; /** Formatter for integers in CompactSize format. */ struct CompactSizeFormatter @@ -656,9 +634,6 @@ class LimitedString } }; -template -BigEndian WrapBigEndian(I& n) { return BigEndian(n); } - /** Formatter to serialize/deserialize vector elements using another formatter * * Example: From 7344c1ae5ff98a75202ce416f58169323290ca5b Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Mon, 30 Mar 2020 15:32:39 -0700 Subject: [PATCH 26/40] Extend CustomUintFormatter to support enums Extracted by Pieter Wuille from a comment by Russ Yanofsky, see https://github.com/bitcoin/bitcoin/pull/18317#discussion_r398821936. --- src/serialize.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index c451b6e30f8e..a68da985ad91 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -567,14 +567,15 @@ struct CustomUintFormatter template void Unser(Stream& s, I& v) { - static_assert(std::numeric_limits::max() >= MAX && std::numeric_limits::min() <= 0, "CustomUintFormatter type too small"); + using U = typename std::conditional::value, std::underlying_type, std::common_type>::type::type; + static_assert(std::numeric_limits::max() >= MAX && std::numeric_limits::min() <= 0, "Assigned type too small"); uint64_t raw = 0; if (BigEndian) { s.read(((char*)&raw) + 8 - Bytes, Bytes); - v = be64toh(raw); + v = static_cast(be64toh(raw)); } else { s.read((char*)&raw, Bytes); - v = le64toh(raw); + v = static_cast(le64toh(raw)); } } }; From 13577fb405c968d0afa796220abb8d24a2bb56fd Mon Sep 17 00:00:00 2001 From: Russell Yanofsky Date: Mon, 30 Mar 2020 15:22:07 -0700 Subject: [PATCH 27/40] Add SER_READ and SER_WRITE for read/write-dependent statements Extracted and extended by Pieter Wuille from a comment by Russ Yanofsky (see https://github.com/bitcoin/bitcoin/pull/18317#discussion_r398625457). --- src/serialize.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index a68da985ad91..55145b6d22be 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -187,6 +187,8 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper(obj))) +#define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const::type& obj) { code; }) +#define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; }) /** * Implement three methods for serializable objects. These are actually wrappers over @@ -1227,6 +1229,28 @@ inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&& ::UnserializeMany(s, args...); } +template +inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&) +{ +} + +template +inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward(obj)); +} + +template +inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward(obj)); +} + +template +inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&) +{ +} + template inline void WriteVarInt(CSizeComputer &s, I n) { From 3d3ee64189aab8ca4b14b6819b11305660dc5bef Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 11 Mar 2020 09:34:20 -0700 Subject: [PATCH 28/40] Convert merkleblock to new serialization --- src/merkleblock.cpp | 18 ++++++++++++++++++ src/merkleblock.h | 39 ++++++++++++--------------------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index fbf6265ed229..d94b919b2988 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -12,6 +12,24 @@ #include "utilstrencodings.h" +std::vector BitsToBytes(const std::vector& bits) +{ + std::vector ret((bits.size() + 7) / 8); + for (unsigned int p = 0; p < bits.size(); p++) { + ret[p / 8] |= bits[p] << (p % 8); + } + return ret; +} + +std::vector BytesToBits(const std::vector& bytes) +{ + std::vector ret(bytes.size() * 8); + for (unsigned int p = 0; p < ret.size(); p++) { + ret[p] = (bytes[p / 8] & (1 << (p % 8))) != 0; + } + return ret; +} + CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter& filter) { header = block.GetBlockHeader(); diff --git a/src/merkleblock.h b/src/merkleblock.h index f60cdb93183f..10565cd0f16a 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -14,6 +14,10 @@ #include +// Helper functions for serialization. +std::vector BitsToBytes(const std::vector& bits); +std::vector BytesToBits(const std::vector& bytes); + /** Data structure that represents a partial merkle tree. * * It represents a subset of the txid's of a known block, in a way that @@ -82,28 +86,15 @@ class CPartialMerkleTree uint256 TraverseAndExtract(int height, unsigned int pos, unsigned int& nBitsUsed, unsigned int& nHashUsed, std::vector& vMatch); public: - /** serialization implementation */ - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CPartialMerkleTree, obj) { - READWRITE(nTransactions); - READWRITE(vHash); - std::vector vBytes; - if (ser_action.ForRead()) { - READWRITE(vBytes); - CPartialMerkleTree& us = *(const_cast(this)); - us.vBits.resize(vBytes.size() * 8); - for (unsigned int p = 0; p < us.vBits.size(); p++) - us.vBits[p] = (vBytes[p / 8] & (1 << (p % 8))) != 0; - us.fBad = false; - } else { - vBytes.resize((vBits.size() + 7) / 8); - for (unsigned int p = 0; p < vBits.size(); p++) - vBytes[p / 8] |= vBits[p] << (p % 8); - READWRITE(vBytes); - } + READWRITE(obj.nTransactions, obj.vHash); + std::vector bytes; + SER_WRITE(obj, bytes = BitsToBytes(obj.vBits)); + READWRITE(bytes); + SER_READ(obj, obj.vBits = BytesToBits(bytes)); + SER_READ(obj, obj.fBad = false); } /** Construct a partial merkle tree from a list of transaction id's, and a mask that selects a subset of them */ @@ -143,14 +134,8 @@ class CMerkleBlock CMerkleBlock() {} - ADD_SERIALIZE_METHODS; + SERIALIZE_METHODS(CMerkleBlock, obj) { READWRITE(obj.header, obj.txn); } - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(header); - READWRITE(txn); - } }; #endif // BITCOIN_MERKLEBLOCK_H From 0f157849c42849eb1eaee86e614740270687b888 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 11 Mar 2020 09:35:50 -0700 Subject: [PATCH 29/40] Convert everything except wallet/qt to new serialization (step 1) --- src/bench/prevector.cpp | 1 + src/bloom.h | 17 ++++----------- src/flatfile.h | 8 +------ src/netaddress.h | 26 +++------------------- src/policy/feerate.h | 8 +------ src/primitives/block.h | 42 +++++++++++++----------------------- src/primitives/transaction.h | 25 +++------------------ src/protocol.h | 42 ++++++++++-------------------------- src/rest.cpp | 9 ++------ src/script/keyorigin.h | 8 +------ src/script/script.h | 7 +----- src/test/dbwrapper_tests.cpp | 36 ++++++++++++++++--------------- src/test/serialize_tests.cpp | 23 +++++++++----------- src/txdb.cpp | 14 +----------- 14 files changed, 73 insertions(+), 193 deletions(-) diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index f192fc370ff6..75bfd7773056 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -22,6 +22,7 @@ struct nontrivial_t { int x; nontrivial_t() :x(-1) {} + SERIALIZE_METHODS(nontrivial_t, obj) { READWRITE(obj.x); } }; static_assert(!IS_TRIVIALLY_CONSTRUCTIBLE::value, "expected nontrivial_t to not be trivially constructible"); diff --git a/src/bloom.h b/src/bloom.h index 6337e6198a85..f16da1cfcac3 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -33,11 +33,11 @@ enum bloomflags { /** * BloomFilter is a probabilistic filter which SPV clients provide * so that we can filter the transactions we sends them. - * + * * This allows for significantly more efficient transaction and block downloads. - * + * * Because bloom filters are probabilistic, an SPV node can increase the false- - * positive rate, making us send them transactions which aren't actually theirs, + * positive rate, making us send them transactions which aren't actually theirs, * allowing clients to trade more bandwidth for more privacy by obfuscating which * keys are owned by them. */ @@ -66,16 +66,7 @@ class CBloomFilter CBloomFilter(unsigned int nElements, double nFPRate, unsigned int nTweak, unsigned char nFlagsIn); CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(vData); - READWRITE(nHashFuncs); - READWRITE(nTweak); - READWRITE(nFlags); - } + SERIALIZE_METHODS(CBloomFilter, obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); } void insert(const std::vector& vKey); void insert(const COutPoint& outpoint); diff --git a/src/flatfile.h b/src/flatfile.h index 2d1e7e41d504..36d5cda6eef1 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -16,13 +16,7 @@ struct FlatFilePos int nFile; unsigned int nPos; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT_MODE(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); - } + SERIALIZE_METHODS(FlatFilePos, obj) { READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos)); } FlatFilePos() : nFile(-1), nPos(0) {} diff --git a/src/netaddress.h b/src/netaddress.h index ea3a2d99ff68..316651dcf3f8 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -90,13 +90,7 @@ class CNetAddr friend bool operator!=(const CNetAddr& a, const CNetAddr& b); friend bool operator<(const CNetAddr& a, const CNetAddr& b); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(ip); - } + SERIALIZE_METHODS(CNetAddr, obj) { READWRITE(obj.ip); } friend class CSubNet; }; @@ -128,15 +122,7 @@ class CSubNet friend bool operator!=(const CSubNet& a, const CSubNet& b); friend bool operator<(const CSubNet& a, const CSubNet& b); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(network); - READWRITE(netmask); - READWRITE(valid); - } + SERIALIZE_METHODS(CSubNet, obj) { READWRITE(obj.network, obj.netmask, obj.valid); } }; /** A combination of a network address (CNetAddr) and a (TCP) port */ @@ -166,13 +152,7 @@ class CService : public CNetAddr CService(const struct in6_addr& ipv6Addr, unsigned short port); CService(const struct sockaddr_in6& addr); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(ip); - READWRITE(Using>(port)); - } + SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using>(obj.port)); } }; #endif // PIVX_NETADDRESS_H diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 4488604a061f..2a3fc5078bd8 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -36,13 +36,7 @@ class CFeeRate CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString() const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(nSatoshisPerK); - } + SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; #endif // PIVX_POLICY_FEERATE_H diff --git a/src/primitives/block.h b/src/primitives/block.h index def939b83a51..597f9ec90fad 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -38,24 +38,16 @@ class CBlockHeader SetNull(); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nVersion); - READWRITE(hashPrevBlock); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); + SERIALIZE_METHODS(CBlockHeader, obj) { + READWRITE(obj.nVersion, obj.hashPrevBlock, obj.hashMerkleRoot, obj.nTime, obj.nBits, obj.nNonce); //zerocoin active, header changes to include accumulator checksum - if(nVersion > 3 && nVersion < 7) - READWRITE(nAccumulatorCheckpoint); + if(obj.nVersion > 3 && obj.nVersion < 7) + READWRITE(obj.nAccumulatorCheckpoint); // Sapling active - if (nVersion >= 8) - READWRITE(hashFinalSaplingRoot); + if (obj.nVersion >= 8) + READWRITE(obj.hashFinalSaplingRoot); } void SetNull() @@ -107,14 +99,12 @@ class CBlock : public CBlockHeader *(static_cast(this)) = header; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CBlockHeader, *this); - READWRITE(vtx); - if(vtx.size() > 1 && vtx[1]->IsCoinStake()) - READWRITE(vchBlockSig); + SERIALIZE_METHODS(CBlock, obj) + { + READWRITEAS(CBlockHeader, obj); + READWRITE(obj.vtx); + if(obj.vtx.size() > 1 && obj.vtx[1]->IsCoinStake()) + READWRITE(obj.vchBlockSig); } void SetNull() @@ -171,14 +161,12 @@ struct CBlockLocator vHave = vHaveIn; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CBlockLocator, obj) + { int nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); - READWRITE(vHave); + READWRITE(obj.vHave); } void SetNull() diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index f0989ca53efc..029a345616de 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -39,13 +39,7 @@ class BaseOutPoint BaseOutPoint(const uint256& hashIn, const uint32_t nIn, bool isTransparentIn = true) : hash(hashIn), n(nIn), isTransparent(isTransparentIn) { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(hash); - READWRITE(n); - } + SERIALIZE_METHODS(BaseOutPoint, obj) { READWRITE(obj.hash, obj.n); } void SetNull() { hash.SetNull(); n = (uint32_t) -1; } bool IsNull() const { return (hash.IsNull() && n == (uint32_t) -1); } @@ -110,14 +104,7 @@ class CTxIn explicit CTxIn(COutPoint prevoutIn, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); CTxIn(uint256 hashPrevTx, uint32_t nOut, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(prevout); - READWRITE(scriptSig); - READWRITE(nSequence); - } + SERIALIZE_METHODS(CTxIn, obj) { READWRITE(obj.prevout, obj.scriptSig, obj.nSequence); } bool IsFinal() const { return nSequence == SEQUENCE_FINAL; } bool IsNull() const { return prevout.IsNull() && scriptSig.empty() && IsFinal(); } @@ -159,13 +146,7 @@ class CTxOut CTxOut(const CAmount& nValueIn, CScript scriptPubKeyIn); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nValue); - READWRITE(scriptPubKey); - } + SERIALIZE_METHODS(CTxOut, obj) { READWRITE(obj.nValue, obj.scriptPubKey); } void SetNull() { diff --git a/src/protocol.h b/src/protocol.h index 6cc540b3341f..0c3143dc79f1 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -43,16 +43,7 @@ class CMessageHeader std::string GetCommand() const; bool IsValid(const MessageStartChars& messageStart) const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(pchMessageStart); - READWRITE(pchCommand); - READWRITE(nMessageSize); - READWRITE(pchChecksum); - } + SERIALIZE_METHODS(CMessageHeader, obj) { READWRITE(obj.pchMessageStart, obj.pchCommand, obj.nMessageSize, obj.pchChecksum); } // TODO: make private (improves encapsulation) public: @@ -319,23 +310,19 @@ class CAddress : public CService void Init(); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CAddress, obj) { - if (ser_action.ForRead()) - Init(); + SER_READ(obj, obj.Init()); int nVersion = s.GetVersion(); - if (s.GetType() & SER_DISK) + if (s.GetType() & SER_DISK) { READWRITE(nVersion); + } if ((s.GetType() & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) - READWRITE(nTime); - uint64_t nServicesInt = nServices; - READWRITE(nServicesInt); - nServices = static_cast(nServicesInt); - READWRITEAS(CService, *this); + (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) { + READWRITE(obj.nTime); + } + READWRITE(Using>(obj.nServices)); + READWRITEAS(CService, obj); } // TODO: make private (improves encapsulation) @@ -353,14 +340,7 @@ class CInv CInv(); CInv(int typeIn, const uint256& hashIn); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(type); - READWRITE(hash); - } + SERIALIZE_METHODS(CInv, obj) { READWRITE(obj.type, obj.hash); } friend bool operator<(const CInv& a, const CInv& b); diff --git a/src/rest.cpp b/src/rest.cpp index 3c15c7cd5704..9b407b4691f9 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -46,18 +46,13 @@ struct CCoin { uint32_t nHeight; CTxOut out; - ADD_SERIALIZE_METHODS; - CCoin() : nHeight(0) {} CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoin, obj) { uint32_t nTxVerDummy = 0; - READWRITE(nTxVerDummy); - READWRITE(nHeight); - READWRITE(out); + READWRITE(nTxVerDummy, obj.nHeight, obj.out); } }; diff --git a/src/script/keyorigin.h b/src/script/keyorigin.h index 949dd8f6426d..bfa0dc214945 100644 --- a/src/script/keyorigin.h +++ b/src/script/keyorigin.h @@ -18,13 +18,7 @@ struct KeyOriginInfo return std::equal(std::begin(a.fingerprint), std::end(a.fingerprint), std::begin(b.fingerprint)) && a.path == b.path; } - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(fingerprint); - READWRITE(path); - } + SERIALIZE_METHODS(KeyOriginInfo, obj) { READWRITE(obj.fingerprint, obj.path); } void clear() { diff --git a/src/script/script.h b/src/script/script.h index bf332f89201c..319342e95b39 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -396,12 +396,7 @@ class CScript : public CScriptBase CScript(std::vector::const_iterator pbegin, std::vector::const_iterator pend) : CScriptBase(pbegin, pend) { } CScript(const unsigned char* pbegin, const unsigned char* pend) : CScriptBase(pbegin, pend) { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CScriptBase, *this); - } + SERIALIZE_METHODS(CScript, obj) { READWRITEAS(CScriptBase, obj); } CScript& operator+=(const CScript& b) { diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 0c2d1a58909b..292dc4a66a6b 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -239,24 +239,26 @@ struct StringContentsSerializer { } StringContentsSerializer& operator+=(const StringContentsSerializer& s) { return *this += s.str; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - if (ser_action.ForRead()) { - str.clear(); - char c = 0; - while (true) { - try { - READWRITE(c); - str.push_back(c); - } catch (const std::ios_base::failure& e) { - break; - } + template + void Serialize(Stream& s) const + { + for (size_t i = 0; i < str.size(); i++) { + s << str[i]; + } + } + + template + void Unserialize(Stream& s) + { + str.clear(); + char c = 0; + while (true) { + try { + s >> c; + str.push_back(c); + } catch (const std::ios_base::failure&) { + break; } - } else { - for (size_t i = 0; i < str.size(); i++) - READWRITE(str[i]); } } }; diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index dc6a3be81cca..481265b6da35 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -48,15 +48,13 @@ class CSerializeMethodsTestSingle memcpy(charstrval, charstrvalin, sizeof(charstrval)); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval); - READWRITE(boolval); - READWRITE(stringval); - READWRITE(charstrval); - READWRITE(txval); + SERIALIZE_METHODS(CSerializeMethodsTestSingle, obj) + { + READWRITE(obj.intval); + READWRITE(obj.boolval); + READWRITE(obj.stringval); + READWRITE(obj.charstrval); + READWRITE(obj.txval); } bool operator==(const CSerializeMethodsTestSingle& rhs) @@ -73,11 +71,10 @@ class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle { public: using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval, boolval, stringval, charstrval, txval); + SERIALIZE_METHODS(CSerializeMethodsTestMany, obj) + { + READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, obj.txval); } }; diff --git a/src/txdb.cpp b/src/txdb.cpp index 469405470bb9..4a3da26bd2cb 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -38,19 +38,7 @@ struct CoinEntry char key; explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} - template - void Serialize(Stream &s) const { - s << key; - s << outpoint->hash; - s << VARINT(outpoint->n); - } - - template - void Unserialize(Stream& s) { - s >> key; - s >> outpoint->hash; - s >> VARINT(outpoint->n); - } + SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); } }; } From eccd4736e41227b78ddedc532fddbfd23642367b Mon Sep 17 00:00:00 2001 From: furszy Date: Tue, 1 Jun 2021 00:02:13 -0300 Subject: [PATCH 30/40] Convert to new serialization (step 2). Focused on object's serializations that doesn't require an special treatment. --- src/budget/budgetproposal.h | 22 ++++----- src/budget/finalizedbudget.h | 30 ++++-------- src/crypter.h | 12 +---- src/evo/deterministicmns.h | 38 ++++++-------- src/libzerocoin/Coin.h | 8 +-- .../CoinRandomnessSchnorrSignature.h | 8 +-- src/libzerocoin/CoinSpend.h | 29 +++-------- src/libzerocoin/Commitment.h | 7 +-- src/libzerocoin/Params.h | 49 ++++++------------- src/masternode-payments.h | 27 ++-------- src/masternode.h | 23 ++++----- src/sapling/address.h | 26 ++-------- src/spork.h | 9 +--- 13 files changed, 78 insertions(+), 210 deletions(-) diff --git a/src/budget/budgetproposal.h b/src/budget/budgetproposal.h index 8919f8b70d70..c907277efebf 100644 --- a/src/budget/budgetproposal.h +++ b/src/budget/budgetproposal.h @@ -106,19 +106,17 @@ class CBudgetProposal } // Serialization for local DB - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CBudgetProposal, obj) { - READWRITE(LIMITED_STRING(strProposalName, 20)); - READWRITE(LIMITED_STRING(strURL, 64)); - READWRITE(nBlockStart); - READWRITE(nBlockEnd); - READWRITE(nAmount); - READWRITE(address); - READWRITE(nFeeTXHash); - READWRITE(nTime); - READWRITE(mapVotes); + READWRITE(LIMITED_STRING(obj.strProposalName, 20)); + READWRITE(LIMITED_STRING(obj.strURL, 64)); + READWRITE(obj.nBlockStart); + READWRITE(obj.nBlockEnd); + READWRITE(obj.nAmount); + READWRITE(obj.address); + READWRITE(obj.nFeeTXHash); + READWRITE(obj.nTime); + READWRITE(obj.mapVotes); } // Serialization for network messages. diff --git a/src/budget/finalizedbudget.h b/src/budget/finalizedbudget.h index 465187a9bbdd..77314659b967 100644 --- a/src/budget/finalizedbudget.h +++ b/src/budget/finalizedbudget.h @@ -106,18 +106,16 @@ class CFinalizedBudget } // Serialization for local DB - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CFinalizedBudget, obj) { - READWRITE(LIMITED_STRING(strBudgetName, 20)); - READWRITE(nFeeTXHash); - READWRITE(nTime); - READWRITE(nBlockStart); - READWRITE(vecBudgetPayments); - READWRITE(fAutoChecked); - READWRITE(mapVotes); - READWRITE(strProposals); + READWRITE(LIMITED_STRING(obj.strBudgetName, 20)); + READWRITE(obj.nFeeTXHash); + READWRITE(obj.nTime); + READWRITE(obj.nBlockStart); + READWRITE(obj.vecBudgetPayments); + READWRITE(obj.fAutoChecked); + READWRITE(obj.mapVotes); + READWRITE(obj.strProposals); } // Serialization for network messages. @@ -155,16 +153,8 @@ class CTxBudgetPayment nAmount(_nAmount) {} - ADD_SERIALIZE_METHODS; - //for saving to the serialized db - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(payee); - READWRITE(nAmount); - READWRITE(nProposalHash); - } + SERIALIZE_METHODS(CTxBudgetPayment, obj) { READWRITE(obj.payee, obj.nAmount, obj.nProposalHash); } // compare payments by proposal hash inline bool operator>(const CTxBudgetPayment& other) const diff --git a/src/crypter.h b/src/crypter.h index c5e4008b6481..6b610fbe5e40 100644 --- a/src/crypter.h +++ b/src/crypter.h @@ -46,17 +46,7 @@ class CMasterKey //! such as the various parameters to scrypt std::vector vchOtherDerivationParameters; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(vchCryptedKey); - READWRITE(vchSalt); - READWRITE(nDerivationMethod); - READWRITE(nDeriveIterations); - READWRITE(vchOtherDerivationParameters); - } + SERIALIZE_METHODS(CMasterKey, obj) { READWRITE(obj.vchCryptedKey, obj.vchSalt, obj.nDerivationMethod, obj.nDeriveIterations, obj.vchOtherDerivationParameters); } CMasterKey() { diff --git a/src/evo/deterministicmns.h b/src/evo/deterministicmns.h index 69363f07d302..79c80e9eafcc 100644 --- a/src/evo/deterministicmns.h +++ b/src/evo/deterministicmns.h @@ -57,30 +57,24 @@ class CDeterministicMNState scriptOperatorPayout = pl.scriptOperatorPayout; } template - CDeterministicMNState(deserialize_type, Stream& s) - { - s >> *this; - } + CDeterministicMNState(deserialize_type, Stream& s) { s >> *this; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDeterministicMNState, obj) { - READWRITE(nRegisteredHeight); - READWRITE(nLastPaidHeight); - READWRITE(nPoSePenalty); - READWRITE(nPoSeRevivedHeight); - READWRITE(nPoSeBanHeight); - READWRITE(nRevocationReason); - READWRITE(confirmedHash); - READWRITE(confirmedHashWithProRegTxHash); - READWRITE(keyIDOwner); - READWRITE(keyIDOperator); - READWRITE(keyIDVoting); - READWRITE(addr); - READWRITE(scriptPayout); - READWRITE(scriptOperatorPayout); + READWRITE(obj.nRegisteredHeight); + READWRITE(obj.nLastPaidHeight); + READWRITE(obj.nPoSePenalty); + READWRITE(obj.nPoSeRevivedHeight); + READWRITE(obj.nPoSeBanHeight); + READWRITE(obj.nRevocationReason); + READWRITE(obj.confirmedHash); + READWRITE(obj.confirmedHashWithProRegTxHash); + READWRITE(obj.keyIDOwner); + READWRITE(obj.keyIDOperator); + READWRITE(obj.keyIDVoting); + READWRITE(obj.addr); + READWRITE(obj.scriptPayout); + READWRITE(obj.scriptOperatorPayout); } void UpdateConfirmedHash(const uint256& _proTxHash, const uint256& _confirmedHash) diff --git a/src/libzerocoin/Coin.h b/src/libzerocoin/Coin.h index 8860d41c48d6..0f72a4e09c98 100644 --- a/src/libzerocoin/Coin.h +++ b/src/libzerocoin/Coin.h @@ -75,13 +75,7 @@ class PublicCoin */ bool validate() const; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(value); - READWRITE(denomination); - } + SERIALIZE_METHODS(PublicCoin, obj) { READWRITE(obj.value, obj.denomination); } private: const ZerocoinParams* params; diff --git a/src/libzerocoin/CoinRandomnessSchnorrSignature.h b/src/libzerocoin/CoinRandomnessSchnorrSignature.h index 8a401a31e52d..58191ec4c0be 100644 --- a/src/libzerocoin/CoinRandomnessSchnorrSignature.h +++ b/src/libzerocoin/CoinRandomnessSchnorrSignature.h @@ -39,13 +39,7 @@ class CoinRandomnessSchnorrSignature { */ bool Verify(const ZerocoinParams* zcparams, const CBigNum& S, const CBigNum& C, const uint256 msghash) const; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(alpha); - READWRITE(beta); - } + SERIALIZE_METHODS(CoinRandomnessSchnorrSignature, obj) { READWRITE(obj.alpha, obj.beta); } private: // signature components diff --git a/src/libzerocoin/CoinSpend.h b/src/libzerocoin/CoinSpend.h index f6540cfc2a6f..a85099fc0a50 100644 --- a/src/libzerocoin/CoinSpend.h +++ b/src/libzerocoin/CoinSpend.h @@ -32,14 +32,11 @@ namespace libzerocoin class AccumulatorProofOfKnowledge { public: AccumulatorProofOfKnowledge() {}; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(C_e); READWRITE(C_u); READWRITE(C_r); READWRITE(st_1); READWRITE(st_2); READWRITE(st_3); - READWRITE(t_1); READWRITE(t_2); READWRITE(t_3); READWRITE(t_4); READWRITE(s_alpha); READWRITE(s_beta); - READWRITE(s_zeta); READWRITE(s_sigma); READWRITE(s_eta); READWRITE(s_epsilon); - READWRITE(s_delta); READWRITE(s_xi); READWRITE(s_phi); READWRITE(s_gamma); READWRITE(s_psi); + SERIALIZE_METHODS(AccumulatorProofOfKnowledge, obj) { + READWRITE(obj.C_e, obj.C_u, obj.C_r, obj.st_1, obj.st_2, obj.st_3); + READWRITE(obj.t_1, obj.t_2, obj.t_3, obj.t_4, obj.s_alpha, obj.s_beta); + READWRITE(obj.s_zeta, obj.s_sigma, obj.s_eta, obj.s_epsilon); + READWRITE(obj.s_delta, obj.s_xi, obj.s_phi, obj.s_gamma, obj.s_psi); } private: CBigNum C_e, C_u, C_r; @@ -55,14 +52,7 @@ class AccumulatorProofOfKnowledge { class SerialNumberSignatureOfKnowledge { public: SerialNumberSignatureOfKnowledge(){}; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(s_notprime); - READWRITE(sprime); - READWRITE(hash); - } + SERIALIZE_METHODS(SerialNumberSignatureOfKnowledge, obj) { READWRITE(obj.s_notprime, obj.sprime, obj.hash); } private: uint256 hash; std::vector s_notprime; @@ -74,12 +64,7 @@ class SerialNumberSignatureOfKnowledge { class CommitmentProofOfKnowledge { public: CommitmentProofOfKnowledge() {}; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(S1); READWRITE(S2); READWRITE(S3); READWRITE(challenge); - } + SERIALIZE_METHODS(CommitmentProofOfKnowledge, obj) { READWRITE(obj.S1,obj.S2, obj.S3, obj.challenge); } private: CBigNum S1, S2, S3, challenge; }; diff --git a/src/libzerocoin/Commitment.h b/src/libzerocoin/Commitment.h index 9f766ed8c02e..7ab24530f2c2 100644 --- a/src/libzerocoin/Commitment.h +++ b/src/libzerocoin/Commitment.h @@ -54,12 +54,7 @@ class Commitment { CBigNum randomness; const CBigNum contents; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(commitmentValue); READWRITE(randomness); READWRITE(contents); - } + SERIALIZE_METHODS(Commitment, obj) { READWRITE(obj.commitmentValue, obj.randomness, obj.contents); } }; } /* namespace libzerocoin */ #endif /* COMMITMENT_H_ */ diff --git a/src/libzerocoin/Params.h b/src/libzerocoin/Params.h index 7e650b99600d..b2ad273a342e 100644 --- a/src/libzerocoin/Params.h +++ b/src/libzerocoin/Params.h @@ -56,14 +56,7 @@ class IntegerGroupParams { */ CBigNum groupOrder; - ADD_SERIALIZE_METHODS; - template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(initialized); - READWRITE(g); - READWRITE(h); - READWRITE(modulus); - READWRITE(groupOrder); - } + SERIALIZE_METHODS(IntegerGroupParams, obj) { READWRITE(obj.initialized, obj.g , obj.h, obj.modulus, obj.groupOrder); } }; class AccumulatorAndProofParams { @@ -134,23 +127,15 @@ class AccumulatorAndProofParams { */ uint32_t k_prime; - /** - * Security parameter. - * The statistical zero-knowledgeness of the accumulator proof. - */ - uint32_t k_dprime; - ADD_SERIALIZE_METHODS; - template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(initialized); - READWRITE(accumulatorModulus); - READWRITE(accumulatorBase); - READWRITE(accumulatorPoKCommitmentGroup); - READWRITE(accumulatorQRNCommitmentGroup); - READWRITE(minCoinValue); - READWRITE(maxCoinValue); - READWRITE(k_prime); - READWRITE(k_dprime); - } + /** + * Security parameter. + * The statistical zero-knowledgeness of the accumulator proof. + */ + uint32_t k_dprime; + SERIALIZE_METHODS(AccumulatorAndProofParams, obj) { + READWRITE(obj.initialized, obj.accumulatorModulus, obj.accumulatorBase, obj.accumulatorPoKCommitmentGroup); + READWRITE(obj.accumulatorQRNCommitmentGroup, obj.minCoinValue, obj.maxCoinValue, obj.k_prime, obj.k_dprime); + } }; class ZerocoinParams { @@ -203,16 +188,10 @@ class ZerocoinParams { * proofs. */ uint32_t zkp_hash_len; - - ADD_SERIALIZE_METHODS; - template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(initialized); - READWRITE(accumulatorParams); - READWRITE(coinCommitmentGroup); - READWRITE(serialNumberSoKCommitmentGroup); - READWRITE(zkp_iterations); - READWRITE(zkp_hash_len); - } + + SERIALIZE_METHODS(ZerocoinParams, obj) { + READWRITE(obj.initialized, obj.accumulatorParams, obj.coinCommitmentGroup, obj.serialNumberSoKCommitmentGroup, obj.zkp_iterations, obj.zkp_hash_len); + } }; } /* namespace libzerocoin */ diff --git a/src/masternode-payments.h b/src/masternode-payments.h index 1995e4f43460..177cbe4d2c5b 100644 --- a/src/masternode-payments.h +++ b/src/masternode-payments.h @@ -80,14 +80,7 @@ class CMasternodePayee nVotes = nVotesIn; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(scriptPubKey); - READWRITE(nVotes); - } + SERIALIZE_METHODS(CMasternodePayee, obj) { READWRITE(obj.scriptPubKey, obj.nVotes); } }; // Keep track of votes for payees from masternodes @@ -152,14 +145,7 @@ class CMasternodeBlockPayees bool IsTransactionValid(const CTransaction& txNew); std::string GetRequiredPaymentsString(); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(nBlockHeight); - READWRITE(vecPayments); - } + SERIALIZE_METHODS(CMasternodeBlockPayees, obj) { READWRITE(obj.nBlockHeight, obj.vecPayments); } }; // for storing the winning payments @@ -290,14 +276,7 @@ class CMasternodePayments void FillBlockPayee(CMutableTransaction& txCoinbase, CMutableTransaction& txCoinstake, const CBlockIndex* pindexPrev, bool fProofOfStake) const; std::string ToString() const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(mapMasternodePayeeVotes); - READWRITE(mapMasternodeBlocks); - } + SERIALIZE_METHODS(CMasternodePayments, obj) { READWRITE(obj.mapMasternodePayeeVotes, obj.mapMasternodeBlocks); } }; diff --git a/src/masternode.h b/src/masternode.h index 55fc24b4d6bd..2c5a26ad473c 100644 --- a/src/masternode.h +++ b/src/masternode.h @@ -281,20 +281,17 @@ class CMasternodeBroadcast : public CMasternode bool Sign(const std::string strSignKey); bool CheckSignature() const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CMasternodeBroadcast, obj) { - READWRITE(vin); - READWRITE(addr); - READWRITE(pubKeyCollateralAddress); - READWRITE(pubKeyMasternode); - READWRITE(vchSig); - READWRITE(sigTime); - READWRITE(protocolVersion); - READWRITE(lastPing); - READWRITE(nMessVersion); + READWRITE(obj.vin); + READWRITE(obj.addr); + READWRITE(obj.pubKeyCollateralAddress); + READWRITE(obj.pubKeyMasternode); + READWRITE(obj.vchSig); + READWRITE(obj.sigTime); + READWRITE(obj.protocolVersion); + READWRITE(obj.lastPing); + READWRITE(obj.nMessVersion); } /// Create Masternode broadcast, needs to be relayed manually after that diff --git a/src/sapling/address.h b/src/sapling/address.h index 6c842d453671..352d78649d9f 100644 --- a/src/sapling/address.h +++ b/src/sapling/address.h @@ -30,13 +30,7 @@ class SaplingPaymentAddress { SaplingPaymentAddress() {} SaplingPaymentAddress(const diversifier_t& _d, const uint256& _pk_d) : d(_d), pk_d(_pk_d) { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(d); - READWRITE(pk_d); - } + SERIALIZE_METHODS(SaplingPaymentAddress, obj) { READWRITE(obj.d, obj.pk_d); } //! Get the 256-bit SHA256d hash of this payment address. uint256 GetHash() const; @@ -68,14 +62,7 @@ class SaplingFullViewingKey { SaplingFullViewingKey() : ak(), nk(), ovk() { } SaplingFullViewingKey(uint256 ak, uint256 nk, uint256 ovk) : ak(ak), nk(nk), ovk(ovk) { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(ak); - READWRITE(nk); - READWRITE(ovk); - } + SERIALIZE_METHODS(SaplingFullViewingKey, obj) { READWRITE(obj.ak, obj.nk, obj.ovk); } //! Get the fingerprint of this full viewing key (as defined in ZIP 32). uint256 GetFingerprint() const; @@ -103,14 +90,7 @@ class SaplingExpandedSpendingKey { SaplingExpandedSpendingKey() : ask(), nsk(), ovk() { } SaplingExpandedSpendingKey(uint256 ask, uint256 nsk, uint256 ovk) : ask(ask), nsk(nsk), ovk(ovk) { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(ask); - READWRITE(nsk); - READWRITE(ovk); - } + SERIALIZE_METHODS(SaplingExpandedSpendingKey, obj) { READWRITE(obj.ask, obj.nsk, obj.ovk); } SaplingFullViewingKey full_viewing_key() const; bool IsNull() { return ask.IsNull() && nsk.IsNull() && ovk.IsNull(); } diff --git a/src/spork.h b/src/spork.h index 2142948830e1..3dfd24f67584 100644 --- a/src/spork.h +++ b/src/spork.h @@ -94,14 +94,7 @@ class CSporkManager public: CSporkManager(); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(mapSporksActive); - // we don't serialize private key to prevent its leakage - } + SERIALIZE_METHODS(CSporkManager, obj) { READWRITE(obj.mapSporksActive); } void Clear(); void LoadSporksFromDB(); From 3f7826eb9121722863fe934ffb6942549b4cca7a Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 19 May 2020 14:30:30 -0700 Subject: [PATCH 31/40] Add comments to CustomUintFormatter --- src/serialize.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index 55145b6d22be..786b3c8fb5df 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -549,6 +549,15 @@ struct VarIntFormatter } }; +/** Serialization wrapper class for custom integers and enums. + * + * It permits specifying the serialized size (1 to 8 bytes) and endianness. + * + * Use the big endian mode for values that are stored in memory in native + * byte order, but serialized in big endian notation. This is only intended + * to implement serializers that are compatible with existing formats, and + * its use is not recommended for new data structures. + */ template struct CustomUintFormatter { From 35fca119eb3d6baa339c8b14d51c123e76f8c5a9 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 20 May 2020 10:16:41 -0700 Subject: [PATCH 32/40] Convert Qt to new serialization --- src/qt/recentrequeststablemodel.h | 19 +++++----------- src/qt/walletmodel.h | 37 ++++++++++++------------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 479471a59509..ab90966ef89a 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -25,20 +25,11 @@ class RecentRequestEntry QDateTime date; SendCoinsRecipient recipient; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - unsigned int nDate = date.toTime_t(); - - READWRITE(nVersion); - READWRITE(id); - READWRITE(nDate); - READWRITE(recipient); - - if (ser_action.ForRead()) - date = QDateTime::fromTime_t(nDate); + SERIALIZE_METHODS(RecentRequestEntry, obj) { + unsigned int date_timet; + SER_WRITE(obj, date_timet = obj.date.toTime_t()); + READWRITE(obj.nVersion, obj.id, date_timet, obj.recipient); + SER_READ(obj, obj.date = QDateTime::fromTime_t(date_timet)); } }; diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index ada0a107703f..4474727deef7 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -85,30 +85,21 @@ class SendCoinsRecipient static const int CURRENT_VERSION = 1; int nVersion; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(SendCoinsRecipient, obj) { - std::string sAddress = address.toStdString(); - std::string sLabel = label.toStdString(); - std::string sMessage = message.toStdString(); - std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); - - READWRITE(this->nVersion); - READWRITE(sAddress); - READWRITE(sLabel); - READWRITE(amount); - READWRITE(sMessage); - READWRITE(sPaymentRequest); - READWRITE(sAuthenticatedMerchant); - - if (ser_action.ForRead()) { - address = QString::fromStdString(sAddress); - label = QString::fromStdString(sLabel); - message = QString::fromStdString(sMessage); - authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); - } + std::string address_str, label_str, message_str, auth_merchant_str; + + SER_WRITE(obj, address_str = obj.address.toStdString()); + SER_WRITE(obj, label_str = obj.label.toStdString()); + SER_WRITE(obj, message_str = obj.message.toStdString()); + SER_WRITE(obj, auth_merchant_str = obj.authenticatedMerchant.toStdString()); + + READWRITE(obj.nVersion, address_str, label_str, obj.amount, message_str, obj.sPaymentRequest, auth_merchant_str); + + SER_READ(obj, obj.address = QString::fromStdString(address_str)); + SER_READ(obj, obj.label = QString::fromStdString(label_str)); + SER_READ(obj, obj.message = QString::fromStdString(message_str)); + SER_READ(obj, obj.authenticatedMerchant = QString::fromStdString(auth_merchant_str)); } }; From dc0fc95cfd1a0d013d25e69ace04c7ad9ed7e39f Mon Sep 17 00:00:00 2001 From: furszy Date: Tue, 1 Jun 2021 18:10:33 -0300 Subject: [PATCH 33/40] Remove old MESS_VER_STRMESS message version try-catch. --- src/budget/budgetvote.h | 7 +------ src/budget/finalizedbudgetvote.h | 7 +------ src/masternode-payments.h | 7 +------ src/masternode.h | 7 +------ src/messagesigner.h | 2 +- src/rpc/budget.cpp | 4 +--- src/spork.h | 7 +------ 7 files changed, 7 insertions(+), 34 deletions(-) diff --git a/src/budget/budgetvote.h b/src/budget/budgetvote.h index b825a5a3d116..a2e38ab9bd2d 100644 --- a/src/budget/budgetvote.h +++ b/src/budget/budgetvote.h @@ -76,12 +76,7 @@ class CBudgetVote : public CSignedMessage nVote = (VoteDirection) nVoteInt; READWRITE(nTime); READWRITE(vchSig); - try - { - READWRITE(nMessVersion); - } catch (...) { - nMessVersion = MessageVersion::MESS_VER_STRMESS; - } + READWRITE(nMessVersion); } }; diff --git a/src/budget/finalizedbudgetvote.h b/src/budget/finalizedbudgetvote.h index 3647bd8ecd35..5ca0ef56b676 100644 --- a/src/budget/finalizedbudgetvote.h +++ b/src/budget/finalizedbudgetvote.h @@ -55,12 +55,7 @@ class CFinalizedBudgetVote : public CSignedMessage READWRITE(nBudgetHash); READWRITE(nTime); READWRITE(vchSig); - try - { - READWRITE(nMessVersion); - } catch (...) { - nMessVersion = MessageVersion::MESS_VER_STRMESS; - } + READWRITE(nMessVersion); } }; diff --git a/src/masternode-payments.h b/src/masternode-payments.h index 177cbe4d2c5b..b0941c6f1036 100644 --- a/src/masternode-payments.h +++ b/src/masternode-payments.h @@ -194,12 +194,7 @@ class CMasternodePaymentWinner : public CSignedMessage READWRITE(nBlockHeight); READWRITE(payee); READWRITE(vchSig); - try - { - READWRITE(nMessVersion); - } catch (...) { - nMessVersion = MessageVersion::MESS_VER_STRMESS; - } + READWRITE(nMessVersion); } std::string ToString() diff --git a/src/masternode.h b/src/masternode.h index 2c5a26ad473c..808696800f21 100644 --- a/src/masternode.h +++ b/src/masternode.h @@ -57,12 +57,7 @@ class CMasternodePing : public CSignedMessage READWRITE(blockHash); READWRITE(sigTime); READWRITE(vchSig); - try - { - READWRITE(nMessVersion); - } catch (...) { - nMessVersion = MessageVersion::MESS_VER_STRMESS; - } + READWRITE(nMessVersion); } uint256 GetHash() const; diff --git a/src/messagesigner.h b/src/messagesigner.h index ded8c6e6e2a2..c3d849c55424 100644 --- a/src/messagesigner.h +++ b/src/messagesigner.h @@ -12,7 +12,7 @@ extern const std::string strMessageMagic; enum MessageVersion { - MESS_VER_STRMESS = 0, + MESS_VER_STRMESS = 0, // old format MESS_VER_HASH = 1, }; diff --git a/src/rpc/budget.cpp b/src/rpc/budget.cpp index 12d30c07a6c2..377f5ae710f6 100644 --- a/src/rpc/budget.cpp +++ b/src/rpc/budget.cpp @@ -742,9 +742,7 @@ UniValue mnbudgetrawvote(const JSONRPCRequest& request) vote.SetVchSig(vchSig); if (!vote.CheckSignature(pmn->pubKeyMasternode.GetID())) { - // try old message version - vote.nMessVersion = MessageVersion::MESS_VER_STRMESS; - if (!vote.CheckSignature(pmn->pubKeyMasternode.GetID())) return "Failure to verify signature."; + return "Failure to verify signature."; } std::string strError; diff --git a/src/spork.h b/src/spork.h index 3dfd24f67584..4dcbae3730f5 100644 --- a/src/spork.h +++ b/src/spork.h @@ -72,12 +72,7 @@ class CSporkMessage : public CSignedMessage READWRITE(nValue); READWRITE(nTimeSigned); READWRITE(vchSig); - try - { - READWRITE(nMessVersion); - } catch (...) { - nMessVersion = MessageVersion::MESS_VER_STRMESS; - } + READWRITE(nMessVersion); } }; From cf06950cf6986f2fe9abee2ef276522d4616dcd5 Mon Sep 17 00:00:00 2001 From: furszy Date: Tue, 1 Jun 2021 18:58:22 -0300 Subject: [PATCH 34/40] Convert to new serialization (step 3) --- src/budget/budgetmanager.h | 25 ++++++---------- src/budget/budgetvote.h | 17 ++--------- src/budget/finalizedbudgetvote.h | 11 +------ src/evo/providertx.h | 43 +++++++++++----------------- src/libzerocoin/CoinSpend.h | 24 ++++------------ src/masternode-payments.h | 12 +------- src/masternode.h | 33 ++++----------------- src/masternodeman.h | 23 +++++++-------- src/sapling/note.h | 24 +++++++--------- src/sapling/sapling_transaction.h | 24 ++-------------- src/sapling/saplingscriptpubkeyman.h | 19 ++++++------ src/sapling/zip32.h | 24 ++-------------- src/spork.h | 12 +------- src/txdb.h | 9 ++---- src/wallet/hdchain.h | 15 +++------- src/zpiv/zpivmodule.h | 31 +++++++------------- 16 files changed, 92 insertions(+), 254 deletions(-) diff --git a/src/budget/budgetmanager.h b/src/budget/budgetmanager.h index eb1cbc3a7f5d..72202db8bfdc 100644 --- a/src/budget/budgetmanager.h +++ b/src/budget/budgetmanager.h @@ -166,30 +166,23 @@ class CBudgetManager // Remove proposal/budget by FeeTx (called when a block is disconnected) void RemoveByFeeTxId(const uint256& feeTxId); - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CBudgetManager, obj) { { - LOCK(cs_proposals); - READWRITE(mapProposals); - READWRITE(mapFeeTxToProposal); + LOCK(obj.cs_proposals); + READWRITE(obj.mapProposals, obj.mapFeeTxToProposal); } { - LOCK(cs_votes); - READWRITE(mapSeenProposalVotes); - READWRITE(mapOrphanProposalVotes); + LOCK(obj.cs_votes); + READWRITE(obj.mapSeenProposalVotes, obj.mapOrphanProposalVotes); } { - LOCK(cs_budgets); - READWRITE(mapFinalizedBudgets); - READWRITE(mapFeeTxToBudget); - READWRITE(mapUnconfirmedFeeTx); + LOCK(obj.cs_budgets); + READWRITE(obj.mapFinalizedBudgets, obj.mapFeeTxToBudget, obj.mapUnconfirmedFeeTx); } { - LOCK(cs_finalizedvotes); - READWRITE(mapSeenFinalizedBudgetVotes); - READWRITE(mapOrphanFinalizedBudgetVotes); + LOCK(obj.cs_finalizedvotes); + READWRITE(obj.mapSeenFinalizedBudgetVotes, obj.mapOrphanFinalizedBudgetVotes); } } }; diff --git a/src/budget/budgetvote.h b/src/budget/budgetvote.h index a2e38ab9bd2d..f50df0e80d62 100644 --- a/src/budget/budgetvote.h +++ b/src/budget/budgetvote.h @@ -17,7 +17,7 @@ class CBudgetVote : public CSignedMessage { public: - enum VoteDirection { + enum VoteDirection : uint32_t { VOTE_ABSTAIN = 0, VOTE_YES = 1, VOTE_NO = 2 @@ -64,20 +64,7 @@ class CBudgetVote : public CSignedMessage void SetTime(const int64_t& _nTime) { nTime = _nTime; } void SetValid(bool _fValid) { fValid = _fValid; } - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(vin); - READWRITE(nProposalHash); - int nVoteInt = (int) nVote; - READWRITE(nVoteInt); - if (ser_action.ForRead()) - nVote = (VoteDirection) nVoteInt; - READWRITE(nTime); - READWRITE(vchSig); - READWRITE(nMessVersion); - } + SERIALIZE_METHODS(CBudgetVote, obj) { READWRITE(obj.vin, obj.nProposalHash, Using>(obj.nVote), obj.nTime, obj.vchSig, obj.nMessVersion); } }; #endif // BUDGET_VOTE_H diff --git a/src/budget/finalizedbudgetvote.h b/src/budget/finalizedbudgetvote.h index 5ca0ef56b676..a446723c7ded 100644 --- a/src/budget/finalizedbudgetvote.h +++ b/src/budget/finalizedbudgetvote.h @@ -47,16 +47,7 @@ class CFinalizedBudgetVote : public CSignedMessage void SetTime(const int64_t& _nTime) { nTime = _nTime; } void SetValid(bool _fValid) { fValid = _fValid; } - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(vin); - READWRITE(nBudgetHash); - READWRITE(nTime); - READWRITE(vchSig); - READWRITE(nMessVersion); - } + SERIALIZE_METHODS(CFinalizedBudgetVote, obj) { READWRITE(obj.vin, obj.nBudgetHash, obj.nTime, obj.vchSig, obj.nMessVersion); } }; #endif // FINALIZED_BUDGET_VOTE_H diff --git a/src/evo/providertx.h b/src/evo/providertx.h index b7e32b3edb40..0ee4cf2d8268 100644 --- a/src/evo/providertx.h +++ b/src/evo/providertx.h @@ -38,25 +38,23 @@ class ProRegPL std::vector vchSig; public: - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(ProRegPL, obj) { - READWRITE(nVersion); - READWRITE(nType); - READWRITE(nMode); - READWRITE(collateralOutpoint); - READWRITE(addr); - READWRITE(keyIDOwner); - READWRITE(keyIDOperator); - READWRITE(keyIDVoting); - READWRITE(scriptPayout); - READWRITE(nOperatorReward); - READWRITE(scriptOperatorPayout); - READWRITE(inputsHash); + READWRITE(obj.nVersion); + READWRITE(obj.nType); + READWRITE(obj.nMode); + READWRITE(obj.collateralOutpoint); + READWRITE(obj.addr); + READWRITE(obj.keyIDOwner); + READWRITE(obj.keyIDOperator); + READWRITE(obj.keyIDVoting); + READWRITE(obj.scriptPayout); + READWRITE(obj.nOperatorReward); + READWRITE(obj.scriptOperatorPayout); + READWRITE(obj.inputsHash); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } } @@ -82,18 +80,11 @@ class ProUpServPL std::vector vchSig; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(ProUpServPL, obj) { - READWRITE(nVersion); - READWRITE(proTxHash); - READWRITE(addr); - READWRITE(scriptOperatorPayout); - READWRITE(inputsHash); + READWRITE(obj.nVersion, obj.proTxHash, obj.addr, obj.scriptOperatorPayout, obj.inputsHash); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } } diff --git a/src/libzerocoin/CoinSpend.h b/src/libzerocoin/CoinSpend.h index a85099fc0a50..5bef6c976ad5 100644 --- a/src/libzerocoin/CoinSpend.h +++ b/src/libzerocoin/CoinSpend.h @@ -107,27 +107,15 @@ class CoinSpend CBigNum CalculateValidSerial(ZerocoinParams* params); std::string ToString() const; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CoinSpend, obj) { - READWRITE(denomination); - READWRITE(ptxHash); - READWRITE(accChecksum); - READWRITE(accCommitmentToCoinValue); - READWRITE(serialCommitmentToCoinValue); - READWRITE(coinSerialNumber); - READWRITE(accumulatorPoK); - READWRITE(serialNumberSoK); - READWRITE(commitmentPoK); - + READWRITE(obj.denomination, obj.ptxHash, obj.accChecksum, obj.accCommitmentToCoinValue); + READWRITE(obj.serialCommitmentToCoinValue, obj.coinSerialNumber, obj.accumulatorPoK); + READWRITE(obj.serialNumberSoK, obj.commitmentPoK); try { - READWRITE(version); - READWRITE(pubkey); - READWRITE(vchSig); - READWRITE(spendType); + READWRITE(obj.version, obj.pubkey, obj.vchSig, obj.spendType); } catch (...) { - version = 1; + SER_READ(obj, obj.version = 1); } } diff --git a/src/masternode-payments.h b/src/masternode-payments.h index b0941c6f1036..187b945fc039 100644 --- a/src/masternode-payments.h +++ b/src/masternode-payments.h @@ -185,17 +185,7 @@ class CMasternodePaymentWinner : public CSignedMessage payee = payeeIn; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(vinMasternode); - READWRITE(nBlockHeight); - READWRITE(payee); - READWRITE(vchSig); - READWRITE(nMessVersion); - } + SERIALIZE_METHODS(CMasternodePaymentWinner, obj) { READWRITE(obj.vinMasternode, obj.nBlockHeight, obj.payee, obj.vchSig, obj.nMessVersion); } std::string ToString() { diff --git a/src/masternode.h b/src/masternode.h index 808696800f21..8dc5812b88f5 100644 --- a/src/masternode.h +++ b/src/masternode.h @@ -48,17 +48,7 @@ class CMasternodePing : public CSignedMessage CMasternodePing(); CMasternodePing(const CTxIn& newVin, const uint256& nBlockHash, uint64_t _sigTime); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(vin); - READWRITE(blockHash); - READWRITE(sigTime); - READWRITE(vchSig); - READWRITE(nMessVersion); - } + SERIALIZE_METHODS(CMasternodePing, obj) { READWRITE(obj.vin, obj.blockHash, obj.sigTime, obj.vchSig, obj.nMessVersion); } uint256 GetHash() const; @@ -153,23 +143,12 @@ class CMasternode : public CSignedMessage arith_uint256 CalculateScore(const uint256& hash) const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CMasternode, obj) { - LOCK(cs); - - READWRITE(vin); - READWRITE(addr); - READWRITE(pubKeyCollateralAddress); - READWRITE(pubKeyMasternode); - READWRITE(vchSig); - READWRITE(sigTime); - READWRITE(protocolVersion); - READWRITE(lastPing); - READWRITE(nScanningErrorCount); - READWRITE(nLastScanningErrorBlockHeight); + LOCK(obj.cs); + READWRITE(obj.vin, obj.addr, obj.pubKeyCollateralAddress); + READWRITE(obj.pubKeyMasternode, obj.vchSig, obj.sigTime, obj.protocolVersion); + READWRITE(obj.lastPing, obj.nScanningErrorCount, obj.nLastScanningErrorBlockHeight); } template diff --git a/src/masternodeman.h b/src/masternodeman.h index cec03ac478a0..cdd6c0ab9335 100644 --- a/src/masternodeman.h +++ b/src/masternodeman.h @@ -93,20 +93,17 @@ class CMasternodeMan // TODO: Remove this from serialization int64_t nDsqCount; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CMasternodeMan, obj) { - LOCK(cs); - READWRITE(mapMasternodes); - READWRITE(mAskedUsForMasternodeList); - READWRITE(mWeAskedForMasternodeList); - READWRITE(mWeAskedForMasternodeListEntry); - READWRITE(nDsqCount); - - READWRITE(mapSeenMasternodeBroadcast); - READWRITE(mapSeenMasternodePing); + LOCK(obj.cs); + READWRITE(obj.mapMasternodes); + READWRITE(obj.mAskedUsForMasternodeList); + READWRITE(obj.mWeAskedForMasternodeList); + READWRITE(obj.mWeAskedForMasternodeListEntry); + READWRITE(obj.nDsqCount); + + READWRITE(obj.mapSeenMasternodeBroadcast); + READWRITE(obj.mapSeenMasternodePing); } CMasternodeMan(); diff --git a/src/sapling/note.h b/src/sapling/note.h index 6081c41702f5..8f552a0ce80b 100644 --- a/src/sapling/note.h +++ b/src/sapling/note.h @@ -89,10 +89,8 @@ class SaplingNotePlaintext : public BaseNotePlaintext { boost::optional note(const SaplingIncomingViewingKey& ivk) const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(SaplingNotePlaintext, obj) + { unsigned char leadingByte = 0x01; READWRITE(leadingByte); @@ -100,10 +98,10 @@ class SaplingNotePlaintext : public BaseNotePlaintext { throw std::ios_base::failure("lead byte of SaplingNotePlaintext is not recognized"); } - READWRITE(d); // 11 bytes - READWRITE(value_); // 8 bytes - READWRITE(rcm); // 32 bytes - READWRITE(memo_); // 512 bytes + READWRITE(obj.d); // 11 bytes + READWRITE(obj.value_); // 8 bytes + READWRITE(obj.rcm); // 32 bytes + READWRITE(obj.memo_); // 512 bytes } boost::optional encrypt(const uint256& pk_d) const; @@ -121,12 +119,10 @@ class SaplingOutgoingPlaintext esk(_esk) {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(pk_d); // 8 bytes - READWRITE(esk); // 8 bytes + SERIALIZE_METHODS(SaplingOutgoingPlaintext, obj) + { + READWRITE(obj.pk_d); // 8 bytes + READWRITE(obj.esk); // 8 bytes } static boost::optional decrypt( diff --git a/src/sapling/sapling_transaction.h b/src/sapling/sapling_transaction.h index a12b7a57f017..250d5e24d35c 100644 --- a/src/sapling/sapling_transaction.h +++ b/src/sapling/sapling_transaction.h @@ -52,17 +52,7 @@ class SpendDescription SpendDescription() {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(cv); - READWRITE(anchor); - READWRITE(nullifier); - READWRITE(rk); - READWRITE(zkproof); - READWRITE(spendAuthSig); - } + SERIALIZE_METHODS(SpendDescription, obj) { READWRITE(obj.cv, obj.anchor, obj.nullifier, obj.rk, obj.zkproof, obj.spendAuthSig); } friend bool operator==(const SpendDescription& a, const SpendDescription& b) { @@ -97,17 +87,7 @@ class OutputDescription OutputDescription() {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(cv); - READWRITE(cmu); - READWRITE(ephemeralKey); - READWRITE(encCiphertext); - READWRITE(outCiphertext); - READWRITE(zkproof); - } + SERIALIZE_METHODS(OutputDescription, obj) { READWRITE(obj.cv, obj.cmu, obj.ephemeralKey, obj.encCiphertext, obj.outCiphertext, obj.zkproof); } friend bool operator==(const OutputDescription& a, const OutputDescription& b) { diff --git a/src/sapling/saplingscriptpubkeyman.h b/src/sapling/saplingscriptpubkeyman.h index 600c8652a6ce..0230eb669622 100644 --- a/src/sapling/saplingscriptpubkeyman.h +++ b/src/sapling/saplingscriptpubkeyman.h @@ -93,22 +93,19 @@ class SaplingNoteData */ Optional nullifier; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(SaplingNoteData, obj) { int nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) { READWRITE(nVersion); } - READWRITE(ivk); - READWRITE(nullifier); - READWRITE(witnesses); - READWRITE(witnessHeight); - READWRITE(amount); - READWRITE(address); - READWRITE(memo); + READWRITE(obj.ivk); + READWRITE(obj.nullifier); + READWRITE(obj.witnesses); + READWRITE(obj.witnessHeight); + READWRITE(obj.amount); + READWRITE(obj.address); + READWRITE(obj.memo); } friend bool operator==(const SaplingNoteData& a, const SaplingNoteData& b) { diff --git a/src/sapling/zip32.h b/src/sapling/zip32.h index 3336ae525093..6312bfb8f24d 100644 --- a/src/sapling/zip32.h +++ b/src/sapling/zip32.h @@ -57,17 +57,7 @@ struct SaplingExtendedFullViewingKey { libzcash::SaplingFullViewingKey fvk; uint256 dk; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(depth); - READWRITE(parentFVKTag); - READWRITE(childIndex); - READWRITE(chaincode); - READWRITE(fvk); - READWRITE(dk); - } + SERIALIZE_METHODS(SaplingExtendedFullViewingKey, obj) { READWRITE(obj.depth, obj.parentFVKTag, obj.childIndex, obj.chaincode, obj.fvk, obj.dk); } boost::optional Derive(uint32_t i) const; @@ -103,17 +93,7 @@ struct SaplingExtendedSpendingKey { libzcash::SaplingExpandedSpendingKey expsk; uint256 dk; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(depth); - READWRITE(parentFVKTag); - READWRITE(childIndex); - READWRITE(chaincode); - READWRITE(expsk); - READWRITE(dk); - } + SERIALIZE_METHODS(SaplingExtendedSpendingKey, obj) { READWRITE(obj.depth, obj.parentFVKTag, obj.childIndex, obj.chaincode, obj.expsk, obj.dk); } static SaplingExtendedSpendingKey Master(const HDSeed& seed); diff --git a/src/spork.h b/src/spork.h index 4dcbae3730f5..a7235cc2ec62 100644 --- a/src/spork.h +++ b/src/spork.h @@ -63,17 +63,7 @@ class CSporkMessage : public CSignedMessage void Relay(); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(nSporkID); - READWRITE(nValue); - READWRITE(nTimeSigned); - READWRITE(vchSig); - READWRITE(nMessVersion); - } + SERIALIZE_METHODS(CSporkMessage, obj) { READWRITE(obj.nSporkID, obj.nValue, obj.nTimeSigned, obj.vchSig, obj.nMessVersion); } }; diff --git a/src/txdb.h b/src/txdb.h index 533d0301c4eb..d3071c5b3fe7 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -44,13 +44,10 @@ struct CDiskTxPos : public FlatFilePos { unsigned int nTxOffset; // after header - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDiskTxPos, obj) { - READWRITEAS(FlatFilePos, *this); - READWRITE(VARINT(nTxOffset)); + READWRITEAS(FlatFilePos, obj); + READWRITE(VARINT(obj.nTxOffset)); } CDiskTxPos(const FlatFilePos& blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) diff --git a/src/wallet/hdchain.h b/src/wallet/hdchain.h index cf8333e6edd2..a29c8dd391f0 100644 --- a/src/wallet/hdchain.h +++ b/src/wallet/hdchain.h @@ -35,21 +35,14 @@ class CHDChain uint32_t nInternalChainCounter{0}; uint32_t nStakingChainCounter{0}; // Chain counter type - uint8_t chainType; + uint8_t chainType{HDChain::ChainCounterType::Standard}; CHDChain(const uint8_t& _chainType = HDChain::ChainCounterType::Standard) : chainType(_chainType) { SetNull(); } - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CHDChain, obj) { - READWRITE(nVersion); - READWRITE(seed_id); - READWRITE(nExternalChainCounter); - READWRITE(nInternalChainCounter); - READWRITE(nStakingChainCounter); - if (nVersion == 1) chainType = HDChain::ChainCounterType::Standard; - else READWRITE(chainType); + READWRITE(obj.nVersion, obj.seed_id, obj.nExternalChainCounter, obj.nInternalChainCounter, obj.nStakingChainCounter); + if (obj.nVersion > 1) READWRITE(obj.chainType); } bool SetNull(); diff --git a/src/zpiv/zpivmodule.h b/src/zpiv/zpivmodule.h index 94025e33af62..78de93b27a1a 100644 --- a/src/zpiv/zpivmodule.h +++ b/src/zpiv/zpivmodule.h @@ -44,29 +44,18 @@ class PublicCoinSpend : public libzerocoin::CoinSpend { unsigned int outputIndex = -1; libzerocoin::PublicCoin pubCoin; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - - READWRITE(version); - - if (version < PUBSPEND_SCHNORR) { - READWRITE(coinSerialNumber); - READWRITE(randomness); - READWRITE(pubkey); - READWRITE(vchSig); - + SERIALIZE_METHODS(PublicCoinSpend, obj) { + READWRITE(obj.version); + if (obj.version < PUBSPEND_SCHNORR) { + READWRITE(obj.coinSerialNumber, obj.randomness, obj.pubkey, obj.vchSig); } else { - READWRITE(coinVersion); - if (coinVersion < libzerocoin::PUBKEY_VERSION) { - READWRITE(coinSerialNumber); - } - else { - READWRITE(pubkey); - READWRITE(vchSig); + READWRITE(obj.coinVersion); + if (obj.coinVersion < libzerocoin::PUBKEY_VERSION) { + READWRITE(obj.coinSerialNumber); + } else { + READWRITE(obj.pubkey, obj.vchSig); } - READWRITE(schnorrSig); + READWRITE(obj.schnorrSig); } } }; From 221bf498842dda8d10222098b7e00175a1c58f5b Mon Sep 17 00:00:00 2001 From: furszy Date: Tue, 1 Jun 2021 20:49:39 -0300 Subject: [PATCH 35/40] Convert wallet to new serialization Adaptation of btc@ef17c03e074b6c3f185afa4eff572ba687c2a171 --- src/wallet/wallet.h | 48 +++++++++++++++++++++++++------------------ src/wallet/walletdb.h | 13 ++++-------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index a05c7062bd08..056583c7e8ea 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -141,29 +141,37 @@ class CKeyPool bool IsExternal() const { return type == HDChain::ChangeType::EXTERNAL; } bool IsStaking() const { return type == HDChain::ChangeType::STAKING; } - ADD_SERIALIZE_METHODS; + template + void Serialize(Stream& s) const + { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) { + s << nVersion; + } + s << nTime << vchPubKey << Span((unsigned char*)&type, 1) << m_pre_split; + } - template - inline void SerializationOp(Stream& s, Operation ser_action) + template + void Unserialize(Stream& s) { int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); - READWRITE(nTime); - READWRITE(vchPubKey); - if (ser_action.ForRead()) { - try { - READWRITE(Span((unsigned char*)&type, 1)); - READWRITE(m_pre_split); - } catch (std::ios_base::failure&) { - /* Set as external address if we can't read the type boolean - (this will be the case for any wallet before the HD chain) */ - type = HDChain::ChangeType::EXTERNAL; - m_pre_split = true; - } - } else { - READWRITE(Span((unsigned char*)&type, 1)); - READWRITE(m_pre_split); + if (!(s.GetType() & SER_GETHASH)) { + s >> nVersion; + } + s >> nTime >> vchPubKey; + try { + s >> Span((unsigned char*)&type, 1); + } catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + type = HDChain::ChangeType::EXTERNAL; + } + try { + s >> m_pre_split; + } catch (std::ios_base::failure&) { + /* flag as pre-split address if we can't read the m_pre_split boolean + (this will be the case for any wallet prior to the HD chain upgrade) */ + m_pre_split = true; } } }; diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 281bba204508..7ca3bc88adb9 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -80,16 +80,11 @@ class CKeyMetadata nCreateTime = nCreateTime_; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CKeyMetadata, obj) { - READWRITE(nVersion); - READWRITE(nCreateTime); - if (HasKeyOrigin()) { - READWRITE(hd_seed_id); - READWRITE(key_origin); + READWRITE(obj.nVersion, obj.nCreateTime); + if (obj.HasKeyOrigin()) { + READWRITE(obj.hd_seed_id, obj.key_origin); } } From 1ee0cb22d3597fc14d37f487c0f179106173b826 Mon Sep 17 00:00:00 2001 From: furszy Date: Wed, 2 Jun 2021 10:57:46 -0300 Subject: [PATCH 36/40] Convert CDiskBlockIndex to new serialization. --- src/chain.h | 120 ++++++++++++++++++++++++---------------------------- 1 file changed, 55 insertions(+), 65 deletions(-) diff --git a/src/chain.h b/src/chain.h index 97cfec01a01d..9b53ed56041c 100644 --- a/src/chain.h +++ b/src/chain.h @@ -276,103 +276,93 @@ class CDiskBlockIndex : public CBlockIndex hashPrev = (pprev ? pprev->GetBlockHash() : UINT256_ZERO); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDiskBlockIndex, obj) { int nSerVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(VARINT_MODE(nSerVersion, VarIntMode::NONNEGATIVE_SIGNED)); - - READWRITE(VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nStatus)); - READWRITE(VARINT(nTx)); - if (nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) - READWRITE(VARINT_MODE(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - if (nStatus & BLOCK_HAVE_DATA) - READWRITE(VARINT(nDataPos)); - if (nStatus & BLOCK_HAVE_UNDO) - READWRITE(VARINT(nUndoPos)); + if (!(s.GetType() & SER_GETHASH)) READWRITE(VARINT_MODE(nSerVersion, VarIntMode::NONNEGATIVE_SIGNED)); + + READWRITE(VARINT_MODE(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT(obj.nStatus)); + READWRITE(VARINT(obj.nTx)); + if (obj.nStatus & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO)) READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED)); + if (obj.nStatus & BLOCK_HAVE_DATA) READWRITE(VARINT(obj.nDataPos)); + if (obj.nStatus & BLOCK_HAVE_UNDO) READWRITE(VARINT(obj.nUndoPos)); if (nSerVersion >= DBI_SER_VERSION_NO_ZC) { // Serialization with CLIENT_VERSION = 4009902+ - READWRITE(nFlags); - READWRITE(this->nVersion); - READWRITE(vStakeModifier); - READWRITE(hashPrev); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); - if(this->nVersion > 3 && this->nVersion < 7) - READWRITE(nAccumulatorCheckpoint); + READWRITE(obj.nFlags); + READWRITE(obj.nVersion); + READWRITE(obj.vStakeModifier); + READWRITE(obj.hashPrev); + READWRITE(obj.hashMerkleRoot); + READWRITE(obj.nTime); + READWRITE(obj.nBits); + READWRITE(obj.nNonce); + if(obj.nVersion > 3 && obj.nVersion < 7) + READWRITE(obj.nAccumulatorCheckpoint); // Sapling blocks - if (this->nVersion >= 8) { - READWRITE(hashFinalSaplingRoot); - READWRITE(nSaplingValue); + if (obj.nVersion >= 8) { + READWRITE(obj.hashFinalSaplingRoot); + READWRITE(obj.nSaplingValue); } - } else if (nSerVersion > DBI_OLD_SER_VERSION && ser_action.ForRead()) { // Serialization with CLIENT_VERSION = 4009901 std::map mapZerocoinSupply; int64_t nMoneySupply = 0; - READWRITE(nMoneySupply); - READWRITE(nFlags); - READWRITE(this->nVersion); - READWRITE(vStakeModifier); - READWRITE(hashPrev); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); - if(this->nVersion > 3) { - READWRITE(mapZerocoinSupply); - if(this->nVersion < 7) READWRITE(nAccumulatorCheckpoint); + SER_READ(obj, nMoneySupply); + READWRITE(obj.nFlags); + READWRITE(obj.nVersion); + READWRITE(obj.vStakeModifier); + READWRITE(obj.hashPrev); + READWRITE(obj.hashMerkleRoot); + READWRITE(obj.nTime); + READWRITE(obj.nBits); + READWRITE(obj.nNonce); + if (obj.nVersion > 3) { + SER_READ(obj, mapZerocoinSupply); + if (obj.nVersion < 7) READWRITE(obj.nAccumulatorCheckpoint); } - } else if (ser_action.ForRead()) { // Serialization with CLIENT_VERSION = 4009900- int64_t nMint = 0; uint256 hashNext{}; int64_t nMoneySupply = 0; - READWRITE(nMint); - READWRITE(nMoneySupply); - READWRITE(nFlags); - if (!Params().GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_V3_4)) { + SER_READ(obj, nMint); + SER_READ(obj, nMoneySupply); + READWRITE(obj.nFlags); + if (!Params().GetConsensus().NetworkUpgradeActive(obj.nHeight, Consensus::UPGRADE_V3_4)) { uint64_t nStakeModifier = 0; READWRITE(nStakeModifier); - this->SetStakeModifier(nStakeModifier, this->GeneratedStakeModifier()); + SER_READ(obj, obj.SetStakeModifier(nStakeModifier, obj.GeneratedStakeModifier())); } else { uint256 nStakeModifierV2; - READWRITE(nStakeModifierV2); - this->SetStakeModifier(nStakeModifierV2); + SER_READ(obj, nStakeModifierV2); + SER_READ(obj, obj.SetStakeModifier(nStakeModifierV2)); } - if (IsProofOfStake()) { + if (obj.IsProofOfStake()) { COutPoint prevoutStake; unsigned int nStakeTime = 0; - READWRITE(prevoutStake); - READWRITE(nStakeTime); + SER_READ(obj, prevoutStake); + SER_READ(obj, nStakeTime); } - READWRITE(this->nVersion); - READWRITE(hashPrev); - READWRITE(hashNext); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); - if(this->nVersion > 3) { + READWRITE(obj.nVersion); + READWRITE(obj.hashPrev); + SER_READ(obj, hashNext); + READWRITE(obj.hashMerkleRoot); + READWRITE(obj.nTime); + READWRITE(obj.nBits); + READWRITE(obj.nNonce); + if (obj.nVersion > 3) { std::map mapZerocoinSupply; std::vector vMintDenominationsInBlock; - READWRITE(nAccumulatorCheckpoint); - READWRITE(mapZerocoinSupply); - READWRITE(vMintDenominationsInBlock); + READWRITE(obj.nAccumulatorCheckpoint); + SER_READ(obj, mapZerocoinSupply); + SER_READ(obj, vMintDenominationsInBlock); } } } - uint256 GetBlockHash() const { CBlockHeader block; From f021897c646f161c74af6aa60b0e99a7caf13786 Mon Sep 17 00:00:00 2001 From: random-zebra Date: Fri, 18 Jun 2021 12:12:24 -0300 Subject: [PATCH 37/40] Fix CDiskBlockIndex serialization of dummy fields for old DB versions --- src/chain.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/chain.h b/src/chain.h index 9b53ed56041c..d2fbe1cb6b0b 100644 --- a/src/chain.h +++ b/src/chain.h @@ -310,7 +310,7 @@ class CDiskBlockIndex : public CBlockIndex // Serialization with CLIENT_VERSION = 4009901 std::map mapZerocoinSupply; int64_t nMoneySupply = 0; - SER_READ(obj, nMoneySupply); + READWRITE(nMoneySupply); READWRITE(obj.nFlags); READWRITE(obj.nVersion); READWRITE(obj.vStakeModifier); @@ -320,7 +320,7 @@ class CDiskBlockIndex : public CBlockIndex READWRITE(obj.nBits); READWRITE(obj.nNonce); if (obj.nVersion > 3) { - SER_READ(obj, mapZerocoinSupply); + READWRITE(mapZerocoinSupply); if (obj.nVersion < 7) READWRITE(obj.nAccumulatorCheckpoint); } } else if (ser_action.ForRead()) { @@ -328,8 +328,8 @@ class CDiskBlockIndex : public CBlockIndex int64_t nMint = 0; uint256 hashNext{}; int64_t nMoneySupply = 0; - SER_READ(obj, nMint); - SER_READ(obj, nMoneySupply); + READWRITE(nMint); + READWRITE(nMoneySupply); READWRITE(obj.nFlags); if (!Params().GetConsensus().NetworkUpgradeActive(obj.nHeight, Consensus::UPGRADE_V3_4)) { uint64_t nStakeModifier = 0; @@ -337,18 +337,18 @@ class CDiskBlockIndex : public CBlockIndex SER_READ(obj, obj.SetStakeModifier(nStakeModifier, obj.GeneratedStakeModifier())); } else { uint256 nStakeModifierV2; - SER_READ(obj, nStakeModifierV2); + READWRITE(nStakeModifierV2); SER_READ(obj, obj.SetStakeModifier(nStakeModifierV2)); } if (obj.IsProofOfStake()) { COutPoint prevoutStake; unsigned int nStakeTime = 0; - SER_READ(obj, prevoutStake); - SER_READ(obj, nStakeTime); + READWRITE(prevoutStake); + READWRITE(nStakeTime); } READWRITE(obj.nVersion); READWRITE(obj.hashPrev); - SER_READ(obj, hashNext); + READWRITE(hashNext); READWRITE(obj.hashMerkleRoot); READWRITE(obj.nTime); READWRITE(obj.nBits); @@ -357,8 +357,8 @@ class CDiskBlockIndex : public CBlockIndex std::map mapZerocoinSupply; std::vector vMintDenominationsInBlock; READWRITE(obj.nAccumulatorCheckpoint); - SER_READ(obj, mapZerocoinSupply); - SER_READ(obj, vMintDenominationsInBlock); + READWRITE(mapZerocoinSupply); + READWRITE(vMintDenominationsInBlock); } } } From 8c74c0911be10f24641653d34f8af9a44402dc2b Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 20 May 2020 11:13:06 -0700 Subject: [PATCH 38/40] Convert LimitedString to formatter --- src/serialize.h | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 786b3c8fb5df..7f94c022db1e 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -532,7 +532,7 @@ static inline Wrapper Using(T&& t) { return Wrapper>(obj) #define VARINT(obj) Using>(obj) #define COMPACTSIZE(obj) Using(obj) -#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) +#define LIMITED_STRING(obj,n) Using>(obj) /** Serialization wrapper class for integers in VarInt format. */ template @@ -616,33 +616,24 @@ struct CompactSizeFormatter } }; -template -class LimitedString +template +struct LimitedStringFormatter { -protected: - std::string& string; - -public: - LimitedString(std::string& _string) : string(_string) {} - - template - void Unserialize(Stream& s) + template + void Unser(Stream& s, std::string& v) { size_t size = ReadCompactSize(s); if (size > Limit) { throw std::ios_base::failure("String length limit exceeded"); } - string.resize(size); - if (size != 0) - s.read((char*)string.data(), size); + v.resize(size); + if (size != 0) s.read((char*)v.data(), size); } - template - void Serialize(Stream& s) const + template + void Ser(Stream& s, const std::string& v) { - WriteCompactSize(s, string.size()); - if (!string.empty()) - s.write((char*)string.data(), string.size()); + s << v; } }; From 060d62bc3d397a8326933c0ad002947321e0b5ce Mon Sep 17 00:00:00 2001 From: furszy Date: Wed, 2 Jun 2021 12:04:23 -0300 Subject: [PATCH 39/40] Convert the last, non-trivial, serialization functions to the new form --- src/evo/deterministicmns.h | 45 ++++++------------ src/sapling/incrementalmerkletree.h | 72 +++++++++++++---------------- src/sapling/sapling_transaction.h | 11 +---- 3 files changed, 47 insertions(+), 81 deletions(-) diff --git a/src/evo/deterministicmns.h b/src/evo/deterministicmns.h index 79c80e9eafcc..14e3e3e959f2 100644 --- a/src/evo/deterministicmns.h +++ b/src/evo/deterministicmns.h @@ -143,13 +143,10 @@ class CDeterministicMNStateDiff #undef DMN_STATE_DIFF_LINE } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDeterministicMNStateDiff, obj) { - READWRITE(VARINT(fields)); -#define DMN_STATE_DIFF_LINE(f) if (fields & Field_##f) READWRITE(state.f); + READWRITE(VARINT(obj.fields)); +#define DMN_STATE_DIFF_LINE(f) if (obj.fields & Field_##f) READWRITE(obj.state.f); DMN_STATE_DIFF_ALL_FIELDS #undef DMN_STATE_DIFF_LINE } @@ -193,22 +190,15 @@ class CDeterministicMN CDeterministicMNStateCPtr pdmnState; public: - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDeterministicMN, obj) { - READWRITE(proTxHash); - READWRITE(VARINT(internalId)); - READWRITE(collateralOutpoint); - READWRITE(nOperatorReward); - READWRITE(pdmnState); + READWRITE(obj.proTxHash); + READWRITE(VARINT(obj.internalId)); + READWRITE(obj.collateralOutpoint); + READWRITE(obj.nOperatorReward); + READWRITE(obj.pdmnState); } - template - void Serialize(Stream& s) const { NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); } - - template - void Unserialize(Stream& s) { SerializationOp(s, CSerActionUnserialize()); } - uint64_t GetInternalId() const; std::string ToString() const; @@ -280,18 +270,12 @@ class CDeterministicMNList { } - template - inline void SerializationOpBase(Stream& s, Operation ser_action) - { - READWRITE(blockHash); - READWRITE(nHeight); - READWRITE(nTotalRegisteredCount); - } - template void Serialize(Stream& s) const { - NCONST_PTR(this)->SerializationOpBase(s, CSerActionSerialize()); + s << blockHash; + s << nHeight; + s << nTotalRegisteredCount; // Serialize the map as a vector WriteCompactSize(s, mnMap.size()); for (const auto& p : mnMap) { @@ -305,8 +289,9 @@ class CDeterministicMNList mnUniquePropertyMap = MnUniquePropertyMap(); mnInternalIdMap = MnInternalIdMap(); - SerializationOpBase(s, CSerActionUnserialize()); - + s >> blockHash; + s >> nHeight; + s >> nTotalRegisteredCount; size_t cnt = ReadCompactSize(s); for (size_t i = 0; i < cnt; i++) { AddMN(std::make_shared(deserialize, s), false); diff --git a/src/sapling/incrementalmerkletree.h b/src/sapling/incrementalmerkletree.h index 2b521b0a5032..b98c564f82a7 100644 --- a/src/sapling/incrementalmerkletree.h +++ b/src/sapling/incrementalmerkletree.h @@ -23,33 +23,35 @@ class MerklePath { std::vector> authentication_path; std::vector index; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + template + void Serialize(Stream &s) const { std::vector> pathBytes; uint64_t indexInt; - if (ser_action.ForRead()) { - READWRITE(pathBytes); - READWRITE(indexInt); - MerklePath &us = *(const_cast(this)); - for (size_t i = 0; i < pathBytes.size(); i++) { - us.authentication_path.push_back(convertBytesVectorToVector(pathBytes[i])); - us.index.push_back((indexInt >> ((pathBytes.size() - 1) - i)) & 1); - } - } else { - assert(authentication_path.size() == index.size()); - pathBytes.resize(authentication_path.size()); - for (size_t i = 0; i < authentication_path.size(); i++) { - pathBytes[i].resize((authentication_path[i].size()+7)/8); - for (unsigned int p = 0; p < authentication_path[i].size(); p++) { - pathBytes[i][p / 8] |= authentication_path[i][p] << (7-(p % 8)); - } + assert(authentication_path.size() == index.size()); + pathBytes.resize(authentication_path.size()); + for (size_t i = 0; i < authentication_path.size(); i++) { + pathBytes[i].resize((authentication_path[i].size()+7)/8); + for (unsigned int p = 0; p < authentication_path[i].size(); p++) { + pathBytes[i][p / 8] |= authentication_path[i][p] << (7-(p % 8)); } - indexInt = convertVectorToInt(index); - READWRITE(pathBytes); - READWRITE(indexInt); + } + indexInt = convertVectorToInt(index); + ::Serialize(s, pathBytes); + ::Serialize(s, indexInt); + } + + template + void Unserialize(Stream &s) + { + std::vector> pathBytes; + uint64_t indexInt; + ::Unserialize(s, pathBytes); + ::Unserialize(s, indexInt); + MerklePath &us = *(const_cast(this)); + for (size_t i = 0; i < pathBytes.size(); i++) { + us.authentication_path.push_back(convertBytesVectorToVector(pathBytes[i])); + us.index.push_back((indexInt >> ((pathBytes.size() - 1) - i)) & 1); } } @@ -110,16 +112,10 @@ friend class IncrementalWitness; return IncrementalWitness(*this); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(IncrementalMerkleTree, obj) { - READWRITE(left); - READWRITE(right); - READWRITE(parents); - - wfcheck(); + READWRITE(obj.left, obj.right, obj.parents); + obj.wfcheck(); } static Hash empty_root() { @@ -181,16 +177,10 @@ friend class IncrementalMerkleTree; void append(Hash obj); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(IncrementalWitness, obj) { - READWRITE(tree); - READWRITE(filled); - READWRITE(cursor); - - cursor_depth = tree.next_depth(filled.size()); + READWRITE(obj.tree, obj.filled, obj.cursor); + SER_READ(obj, obj.cursor_depth = obj.tree.next_depth(obj.filled.size())); } template diff --git a/src/sapling/sapling_transaction.h b/src/sapling/sapling_transaction.h index 250d5e24d35c..d19aabac2e40 100644 --- a/src/sapling/sapling_transaction.h +++ b/src/sapling/sapling_transaction.h @@ -117,16 +117,7 @@ class SaplingTxData std::vector vShieldedOutput; binding_sig_t bindingSig = {{0}}; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(*const_cast(&valueBalance)); - READWRITE(*const_cast*>(&vShieldedSpend)); - READWRITE(*const_cast*>(&vShieldedOutput)); - READWRITE(*const_cast(&bindingSig)); - } + SERIALIZE_METHODS(SaplingTxData, obj) { READWRITE(obj.valueBalance, obj.vShieldedSpend, obj.vShieldedOutput, obj.bindingSig); } explicit SaplingTxData() : valueBalance(0), vShieldedSpend(), vShieldedOutput() { } explicit SaplingTxData(const SaplingTxData& from) : valueBalance(from.valueBalance), vShieldedSpend(from.vShieldedSpend), vShieldedOutput(from.vShieldedOutput), bindingSig(from.bindingSig) {} From bd4b846d26d41ddf4ee61316dcd321316e2fa86f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 20 May 2020 11:13:25 -0700 Subject: [PATCH 40/40] Remove old serialization primitives --- src/serialize.h | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index 7f94c022db1e..6ae8b3b533d3 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -52,26 +52,6 @@ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; struct deserialize_type {}; constexpr deserialize_type deserialize {}; -/** - * Used to bypass the rule against non-const reference to temporary - * where it makes sense with wrappers. - */ -template -inline T& REF(const T& val) -{ - return const_cast(val); -} - -/** - * Used to acquire a non-const pointer "this" to generate bodies - * of const serialization operations from a template - */ -template -inline T* NCONST_PTR(const T* val) -{ - return const_cast(val); -} - //! Safely convert odd char pointer types to standard ones. inline char* CharCast(char* c) { return c; } inline char* CharCast(unsigned char* c) { return (char*)c; } @@ -190,24 +170,6 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } #define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const::type& obj) { code; }) #define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; }) -/** - * Implement three methods for serializable objects. These are actually wrappers over - * "SerializationOp" template, which implements the body of each class' serialization - * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be - * added as members. - */ -#define ADD_SERIALIZE_METHODS \ - template \ - void Serialize(Stream& s) const \ - { \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ - } \ - template \ - void Unserialize(Stream& s) \ - { \ - SerializationOp(s, CSerActionUnserialize()); \ - } - /** * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). * @@ -1136,7 +1098,7 @@ void Unserialize(Stream& is, std::shared_ptr& p) } /** - * Support for ADD_SERIALIZE_METHODS and READWRITE macro + * Support for SERIALIZE_METHODS and READWRITE macro. */ struct CSerActionSerialize { constexpr bool ForRead() const { return false; }