diff --git a/CMakeLists.txt b/CMakeLists.txt index 8ccff3d1251e..019c05b52047 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,6 +355,9 @@ target_include_directories(ZEROCOIN_A PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) set(COMMON_SOURCES ./src/base58.cpp ./src/bip38.cpp + ./src/bls/bls.cpp + ./src/bls/bls_ies.cpp + ./src/bls/bls_worker.cpp ./src/consensus/params.cpp ./src/consensus/upgrades.cpp ./src/chainparams.cpp diff --git a/depends/README.md b/depends/README.md index 5baf70979ada..82f5e2f2a225 100644 --- a/depends/README.md +++ b/depends/README.md @@ -43,7 +43,7 @@ No other options are needed, the paths are automatically configured. Common linux dependencies: - sudo apt-get install make automake cmake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch + sudo apt-get install make automake curl g++-multilib libtool binutils-gold bsdmainutils pkg-config python3 patch For linux ARM cross compilation: diff --git a/depends/packages/bls-dash.mk b/depends/packages/bls-dash.mk new file mode 100644 index 000000000000..eb925775d55c --- /dev/null +++ b/depends/packages/bls-dash.mk @@ -0,0 +1,47 @@ +package=bls-dash +$(package)_version=1.0.0 +# It's actually from https://github.com/Chia-Network/bls-signatures, but we have so many patches atm that it's forked +$(package)_download_path=https://github.com/PIVX-Project/bls-signatures/archive +$(package)_file_name=$($(package)_version).tar.gz +$(package)_sha256_hash=ecbc51457dd3de153af27333038f149ba585e16fe46351b763938cb6ef0f2d9a +$(package)_dependencies=gmp cmake libsodium + +define $(package)_set_vars + $(package)_config_opts=-DCMAKE_INSTALL_PREFIX=$($(package)_staging_dir)/$(host_prefix) + $(package)_config_opts+= -DCMAKE_PREFIX_PATH=$($(package)_staging_dir)/$(host_prefix) + $(package)_config_opts+= -DSTLIB=ON -DSHLIB=OFF -DSTBIN=ON + $(package)_config_opts+= -DBUILD_BLS_PYTHON_BINDINGS=0 -DBUILD_BLS_TESTS=0 -DBUILD_BLS_BENCHMARKS=0 + $(package)_config_opts_linux=-DOPSYS=LINUX -DCMAKE_SYSTEM_NAME=Linux + $(package)_config_opts_darwin=-DOPSYS=MACOSX -DCMAKE_SYSTEM_NAME=Darwin + $(package)_config_opts_mingw32=-DOPSYS=WINDOWS -DCMAKE_SYSTEM_NAME=Windows -DCMAKE_SHARED_LIBRARY_LINK_C_FLAGS="" + $(package)_config_opts_i686+= -DWSIZE=32 + $(package)_config_opts_x86_64+= -DWSIZE=64 + $(package)_config_opts_arm+= -DWSIZE=32 + $(package)_config_opts_armv7l+= -DWSIZE=32 + $(package)_config_opts_debug=-DDEBUG=ON -DCMAKE_BUILD_TYPE=Debug + + ifneq ($(darwin_native_toolchain),) + $(package)_config_opts_darwin+= -DCMAKE_AR="$(host_prefix)/native/bin/$($(package)_ar)" + $(package)_config_opts_darwin+= -DCMAKE_RANLIB="$(host_prefix)/native/bin/$($(package)_ranlib)" + endif +endef + +define $(package)_config_cmds + export CC="$($(package)_cc)" && \ + export CXX="$($(package)_cxx)" && \ + export CFLAGS="$($(package)_cflags) $($(package)_cppflags)" && \ + export CXXFLAGS="$($(package)_cxxflags) $($(package)_cppflags)" && \ + export LDFLAGS="$($(package)_ldflags)" && \ + mkdir -p build && cd build && \ + $(host_prefix)/bin/cmake ../ $($(package)_config_opts) +endef + +define $(package)_build_cmds + cd build && \ + $(MAKE) $($(package)_build_opts) +endef + +define $(package)_stage_cmds + cd build && \ + $(MAKE) install +endef \ No newline at end of file diff --git a/depends/packages/cmake.mk b/depends/packages/cmake.mk new file mode 100644 index 000000000000..959f55faa1ec --- /dev/null +++ b/depends/packages/cmake.mk @@ -0,0 +1,17 @@ +package=cmake +$(package)_version=3.14.7 +$(package)_download_path=https://cmake.org/files/v3.14/ +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_sha256_hash=9221993e0af3e6d10124d840ff24f5b2f3b884416fca04d3312cb0388dec1385 + +define $(package)_config_cmds + ./bootstrap --prefix=$(host_prefix) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef \ No newline at end of file diff --git a/depends/packages/native_cdrkit.mk b/depends/packages/native_cdrkit.mk index 8243458ec858..1b33e6ebd100 100644 --- a/depends/packages/native_cdrkit.mk +++ b/depends/packages/native_cdrkit.mk @@ -4,13 +4,14 @@ $(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c $(package)_file_name=cdrkit-$($(package)_version).tar.bz2 $(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564 $(package)_patches=cdrkit-deterministic.patch +$(package)_dependencies=cmake define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch endef define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) + $(host_prefix)/bin/cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix) endef define $(package)_build_cmds diff --git a/depends/packages/native_libdmg-hfsplus.mk b/depends/packages/native_libdmg-hfsplus.mk index 8493f1d9793d..3013a65c8805 100644 --- a/depends/packages/native_libdmg-hfsplus.mk +++ b/depends/packages/native_libdmg-hfsplus.mk @@ -5,6 +5,7 @@ $(package)_file_name=$($(package)_version).tar.gz $(package)_sha256_hash=56fbdc48ec110966342f0ecddd6f8f89202f4143ed2a3336e42bbf88f940850c $(package)_build_subdir=build $(package)_patches=remove-libcrypto-dependency.patch +$(package)_dependencies=cmake define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/remove-libcrypto-dependency.patch && \ @@ -12,7 +13,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - cmake -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix) .. + $(host_prefix)/bin/cmake -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix)/bin .. endef define $(package)_build_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 41349501135c..e9edc60f1437 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -72,7 +72,7 @@ rust_crates := \ crate_zcash_proofs rust_packages := rust $(rust_crates) -packages:=boost libevent gmp $(zcash_packages) libsodium +packages:=boost libevent gmp $(zcash_packages) libsodium bls-dash cmake qt_packages = qrencode zlib diff --git a/src/Makefile.am b/src/Makefile.am index ad88a3e7e537..d5ae5555ff62 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -26,6 +26,8 @@ BITCOIN_INCLUDES += -I$(srcdir)/rust/include BITCOIN_INCLUDES += -I$(srcdir)/secp256k1/include BITCOIN_INCLUDES += $(UNIVALUE_CFLAGS) +BLS_LIBS=-lbls-dash -lgmp + LIBBITCOIN_SERVER=libbitcoin_server.a LIBBITCOIN_COMMON=libbitcoin_common.a LIBBITCOIN_CLI=libbitcoin_cli.a @@ -154,6 +156,9 @@ BITCOIN_CORE_H = \ bip38.h \ bloom.h \ blocksignature.h \ + bls/bls.h \ + bls/bls_ies.h \ + bls/bls_worker.h \ chain.h \ chainparams.h \ chainparamsbase.h \ @@ -181,6 +186,7 @@ BITCOIN_CORE_H = \ core_io.h \ cuckoocache.h \ crypter.h \ + ctpl.h \ cyclingvector.h \ evo/deterministicmns.h \ evo/evodb.h \ @@ -267,6 +273,8 @@ BITCOIN_CORE_H = \ stakeinput.h \ script/ismine.h \ streams.h \ + support/allocators/mt_pooled_secure.h \ + support/allocators/pooled_secure.h \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ support/cleanse.h \ @@ -493,6 +501,9 @@ libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ base58.cpp \ bip38.cpp \ + bls/bls.cpp \ + bls/bls_ies.cpp \ + bls/bls_worker.cpp \ chainparams.cpp \ consensus/upgrades.cpp \ coins.cpp \ @@ -623,7 +634,7 @@ pivxd_LDADD = \ $(LIBRUSTZCASH) \ $(LIBZCASH_LIBS) -pivxd_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) +pivxd_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(ZMQ_LIBS) $(BLS_LIBS) # pivx-cli binary # pivx_cli_SOURCES = pivx-cli.cpp @@ -644,7 +655,8 @@ pivx_cli_LDADD = \ $(LIBRUSTZCASH) \ $(LIBZCASH_LIBS) -pivx_cli_LDADD += $(BOOST_LIBS) $(EVENT_LIBS) +pivx_cli_LDADD += $(BOOST_LIBS) $(EVENT_LIBS) $(BLS_LIBS) + # # pivx-tx binary # @@ -668,7 +680,8 @@ pivx_tx_LDADD = \ $(LIBRUSTZCASH) \ $(LIBZCASH_LIBS) -pivx_tx_LDADD += $(BOOST_LIBS) +pivx_tx_LDADD += $(BOOST_LIBS) $(BLS_LIBS) + # # bitcoinconsensus library # @@ -696,7 +709,7 @@ if GLIBC_BACK_COMPAT endif libbitcoinconsensus_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) -libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) +libbitcoinconsensus_la_LIBADD = $(LIBSECP256K1) $(BLS_LIBS) libbitcoinconsensus_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL libbitcoinconsensus_la_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 80c81c30276b..042ac3014ac3 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -13,10 +13,13 @@ bench_bench_pivx_SOURCES = \ bench/bench.h \ bench/Examples.cpp \ bench/base58.cpp \ + bench/bls.cpp \ + bench/bls_dkg.cpp \ bench/checkblock.cpp \ bench/checkqueue.cpp \ bench/chacha20.cpp \ bench/crypto_hash.cpp \ + bench/ecdsa.cpp \ bench/lockedpool.cpp \ bench/perf.cpp \ bench/perf.h \ @@ -47,7 +50,7 @@ if ENABLE_ZMQ bench_bench_pivx_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif -bench_bench_pivx_LDADD += $(LIBBITCOIN_CONSENSUS) $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) +bench_bench_pivx_LDADD += $(LIBBITCOIN_CONSENSUS) $(BOOST_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(BLS_LIBS) bench_bench_pivx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) # !TODO: .raw.h generated test files are not removed with make clean diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 0c0e91983ae3..a4bf2c33ba6d 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -647,7 +647,7 @@ qt_pivx_qt_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS) endif qt_pivx_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBBITCOIN_ZEROCOIN) $(LIBSAPLING) $(LIBRUSTZCASH) $(LIBZCASH_LIBS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) \ $(BOOST_LIBS) $(QT_LIBS) $(QT_DBUS_LIBS) $(QR_LIBS) $(SVG_LIBS) $(CHARTS_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(BLS_LIBS) qt_pivx_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_pivx_qt_LIBTOOLFLAGS = $(AM_LIBTOOLFLAGS) --tag CXX diff --git a/src/Makefile.qttest.include b/src/Makefile.qttest.include index b80ec1fa5e6a..4c305cd4f59c 100644 --- a/src/Makefile.qttest.include +++ b/src/Makefile.qttest.include @@ -29,7 +29,7 @@ endif qt_test_test_pivx_qt_LDADD += $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBBITCOIN_ZEROCOIN) $(LIBLEVELDB) $(LIBSAPLING) $(LIBRUSTZCASH) $(LIBZCASH_LIBS) \ $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BOOST_LIBS) $(QT_DBUS_LIBS) $(QT_TEST_LIBS) $(QT_LIBS) \ $(QR_LIBS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBSECP256K1) \ - $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) + $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(BLS_LIBS) qt_test_test_pivx_qt_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(QT_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) qt_test_test_pivx_qt_CXXFLAGS = $(AM_CXXFLAGS) $(QT_PIE_FLAGS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 58fc2b8b1b2d..6b19ad45c6ed 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -94,6 +94,7 @@ BITCOIN_TESTS =\ test/base64_tests.cpp \ test/bech32_tests.cpp \ test/bip32_tests.cpp \ + test/bls_tests.cpp \ test/budget_tests.cpp \ test/checkblock_tests.cpp \ test/Checkpoints_tests.cpp \ @@ -185,7 +186,7 @@ endif test_test_pivx_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) -test_test_pivx_LDADD += $(LIBRUSTZCASH) $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBZCASH_LIBS) +test_test_pivx_LDADD += $(LIBRUSTZCASH) $(LIBBITCOIN_CONSENSUS) $(BDB_LIBS) $(MINIUPNPC_LIBS) $(LIBZCASH_LIBS) $(BLS_LIBS) test_test_pivx_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static if ENABLE_ZMQ diff --git a/src/bench/bench_pivx.cpp b/src/bench/bench_pivx.cpp index aa3ac218b479..d43a81accd44 100644 --- a/src/bench/bench_pivx.cpp +++ b/src/bench/bench_pivx.cpp @@ -5,17 +5,30 @@ #include "bench.h" +#include "bls/bls.h" #include "key.h" +#include "random.h" #include "util/system.h" -int -main(int argc, char** argv) +void InitBLSTests(); +void CleanupBLSTests(); +void CleanupBLSDkgTests(); + +int main(int argc, char** argv) { ECC_Start(); + ECCVerifyHandle globalVerifyHandle; + RandomInit(); + BLSInit(); + InitBLSTests(); SetupEnvironment(); g_logger->m_print_to_file = false; // don't want to write to debug.log file benchmark::BenchRunner::RunAll(); + // need to be called before global destructors kick in (PoolAllocator is needed due to many BLSSecretKeys) + CleanupBLSDkgTests(); + CleanupBLSTests(); + ECC_Stop(); } diff --git a/src/bench/bls.cpp b/src/bench/bls.cpp new file mode 100644 index 000000000000..6736809bbb95 --- /dev/null +++ b/src/bench/bls.cpp @@ -0,0 +1,363 @@ +// Copyright (c) 2018 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" +#include "random.h" +#include "bls/bls_worker.h" +#include "utiltime.h" + +#include + +CBLSWorker blsWorker; + +void InitBLSTests() +{ + blsWorker.Start(); +} + +void CleanupBLSTests() +{ + blsWorker.Stop(); +} + +static void BuildTestVectors(size_t count, size_t invalidCount, + BLSPublicKeyVector& pubKeys, BLSSecretKeyVector& secKeys, BLSSignatureVector& sigs, + std::vector& msgHashes, + std::vector& invalid) +{ + secKeys.resize(count); + pubKeys.resize(count); + sigs.resize(count); + msgHashes.resize(count); + + invalid.resize(count); + for (size_t i = 0; i < invalidCount; i++) { + invalid[i] = true; + } + std::random_shuffle(invalid.begin(), invalid.end()); + + for (size_t i = 0; i < count; i++) { + secKeys[i].MakeNewKey(); + pubKeys[i] = secKeys[i].GetPublicKey(); + msgHashes[i] = GetRandHash(); + sigs[i] = secKeys[i].Sign(msgHashes[i]); + + if (invalid[i]) { + CBLSSecretKey s; + s.MakeNewKey(); + sigs[i] = s.Sign(msgHashes[i]); + } + } +} + +static void BLSPubKeyAggregate_Normal(benchmark::State& state) +{ + CBLSSecretKey secKey1, secKey2; + secKey1.MakeNewKey(); + secKey2.MakeNewKey(); + CBLSPublicKey pubKey1 = secKey1.GetPublicKey(); + CBLSPublicKey pubKey2 = secKey2.GetPublicKey(); + + // Benchmark. + while (state.KeepRunning()) { + CBLSPublicKey k(pubKey1); + k.AggregateInsecure(pubKey2); + } +} + +static void BLSSecKeyAggregate_Normal(benchmark::State& state) +{ + CBLSSecretKey secKey1, secKey2; + secKey1.MakeNewKey(); + secKey2.MakeNewKey(); + CBLSPublicKey pubKey1 = secKey1.GetPublicKey(); + CBLSPublicKey pubKey2 = secKey2.GetPublicKey(); + + // Benchmark. + while (state.KeepRunning()) { + CBLSSecretKey k(secKey1); + k.AggregateInsecure(secKey2); + } +} + +static void BLSSign_Normal(benchmark::State& state) +{ + CBLSSecretKey secKey; + secKey.MakeNewKey(); + CBLSPublicKey pubKey = secKey.GetPublicKey(); + + // Benchmark. + while (state.KeepRunning()) { + uint256 hash = GetRandHash(); + secKey.Sign(hash); + } +} + +static void BLSVerify_Normal(benchmark::State& state) +{ + BLSPublicKeyVector pubKeys; + BLSSecretKeyVector secKeys; + BLSSignatureVector sigs; + std::vector msgHashes; + std::vector invalid; + BuildTestVectors(1000, 10, pubKeys, secKeys, sigs, msgHashes, invalid); + + // Benchmark. + size_t i = 0; + while (state.KeepRunning()) { + bool valid = sigs[i].VerifyInsecure(pubKeys[i], msgHashes[i]); + if (valid && invalid[i]) { + std::cout << "expected invalid but it is valid" << std::endl; + assert(false); + } else if (!valid && !invalid[i]) { + std::cout << "expected valid but it is invalid" << std::endl; + assert(false); + } + i = (i + 1) % pubKeys.size(); + } +} + + +static void BLSVerify_LargeBlock(size_t txCount, benchmark::State& state) +{ + BLSPublicKeyVector pubKeys; + BLSSecretKeyVector secKeys; + BLSSignatureVector sigs; + std::vector msgHashes; + std::vector invalid; + BuildTestVectors(txCount, 0, pubKeys, secKeys, sigs, msgHashes, invalid); + + // Benchmark. + while (state.KeepRunning()) { + for (size_t i = 0; i < pubKeys.size(); i++) { + sigs[i].VerifyInsecure(pubKeys[i], msgHashes[i]); + } + } +} + +static void BLSVerify_LargeBlock1000(benchmark::State& state) +{ + BLSVerify_LargeBlock(1000, state); +} + +static void BLSVerify_LargeBlock10000(benchmark::State& state) +{ + BLSVerify_LargeBlock(10000, state); +} + +static void BLSVerify_LargeBlockSelfAggregated(size_t txCount, benchmark::State& state) +{ + BLSPublicKeyVector pubKeys; + BLSSecretKeyVector secKeys; + BLSSignatureVector sigs; + std::vector msgHashes; + std::vector invalid; + BuildTestVectors(txCount, 0, pubKeys, secKeys, sigs, msgHashes, invalid); + + // Benchmark. + while (state.KeepRunning()) { + CBLSSignature aggSig = CBLSSignature::AggregateInsecure(sigs); + aggSig.VerifyInsecureAggregated(pubKeys, msgHashes); + } +} + +static void BLSVerify_LargeBlockSelfAggregated1000(benchmark::State& state) +{ + BLSVerify_LargeBlockSelfAggregated(1000, state); +} + +static void BLSVerify_LargeBlockSelfAggregated10000(benchmark::State& state) +{ + BLSVerify_LargeBlockSelfAggregated(10000, state); +} + +static void BLSVerify_LargeAggregatedBlock(size_t txCount, benchmark::State& state) +{ + BLSPublicKeyVector pubKeys; + BLSSecretKeyVector secKeys; + BLSSignatureVector sigs; + std::vector msgHashes; + std::vector invalid; + BuildTestVectors(txCount, 0, pubKeys, secKeys, sigs, msgHashes, invalid); + + CBLSSignature aggSig = CBLSSignature::AggregateInsecure(sigs); + + // Benchmark. + while (state.KeepRunning()) { + aggSig.VerifyInsecureAggregated(pubKeys, msgHashes); + } +} + +static void BLSVerify_LargeAggregatedBlock1000(benchmark::State& state) +{ + BLSVerify_LargeAggregatedBlock(1000, state); +} + +static void BLSVerify_LargeAggregatedBlock10000(benchmark::State& state) +{ + BLSVerify_LargeAggregatedBlock(10000, state); +} + +static void BLSVerify_LargeAggregatedBlock1000PreVerified(benchmark::State& state) +{ + BLSPublicKeyVector pubKeys; + BLSSecretKeyVector secKeys; + BLSSignatureVector sigs; + std::vector msgHashes; + std::vector invalid; + BuildTestVectors(1000, 0, pubKeys, secKeys, sigs, msgHashes, invalid); + + CBLSSignature aggSig = CBLSSignature::AggregateInsecure(sigs); + + std::set prevalidated; + + while (prevalidated.size() < 900) { + int idx = GetRandInt((int)pubKeys.size()); + if (prevalidated.count((size_t)idx)) { + continue; + } + prevalidated.emplace((size_t)idx); + } + + // Benchmark. + while (state.KeepRunning()) { + BLSPublicKeyVector nonvalidatedPubKeys; + std::vector nonvalidatedHashes; + nonvalidatedPubKeys.reserve(pubKeys.size()); + nonvalidatedHashes.reserve(msgHashes.size()); + + for (size_t i = 0; i < sigs.size(); i++) { + if (prevalidated.count(i)) { + continue; + } + nonvalidatedPubKeys.emplace_back(pubKeys[i]); + nonvalidatedHashes.emplace_back(msgHashes[i]); + } + + CBLSSignature aggSigCopy = aggSig; + for (auto idx : prevalidated) { + aggSigCopy.SubInsecure(sigs[idx]); + } + + bool valid = aggSigCopy.VerifyInsecureAggregated(nonvalidatedPubKeys, nonvalidatedHashes); + assert(valid); + } +} + +static void BLSVerify_Batched(benchmark::State& state) +{ + BLSPublicKeyVector pubKeys; + BLSSecretKeyVector secKeys; + BLSSignatureVector sigs; + std::vector msgHashes; + std::vector invalid; + BuildTestVectors(1000, 10, pubKeys, secKeys, sigs, msgHashes, invalid); + + // Benchmark. + size_t i = 0; + size_t j = 0; + size_t batchSize = 16; + while (state.KeepRunning()) { + j++; + if ((j % batchSize) != 0) { + continue; + } + + BLSPublicKeyVector testPubKeys; + BLSSignatureVector testSigs; + std::vector testMsgHashes; + testPubKeys.reserve(batchSize); + testSigs.reserve(batchSize); + testMsgHashes.reserve(batchSize); + size_t startI = i; + for (size_t k = 0; k < batchSize; k++) { + testPubKeys.emplace_back(pubKeys[i]); + testSigs.emplace_back(sigs[i]); + testMsgHashes.emplace_back(msgHashes[i]); + i = (i + 1) % pubKeys.size(); + } + + CBLSSignature batchSig = CBLSSignature::AggregateInsecure(testSigs); + bool batchValid = batchSig.VerifyInsecureAggregated(testPubKeys, testMsgHashes); + std::vector valid; + if (batchValid) { + valid.assign(batchSize, true); + } else { + for (size_t k = 0; k < batchSize; k++) { + bool valid1 = testSigs[k].VerifyInsecure(testPubKeys[k], testMsgHashes[k]); + valid.emplace_back(valid1); + } + } + for (size_t k = 0; k < batchSize; k++) { + if (valid[k] && invalid[(startI + k) % pubKeys.size()]) { + std::cout << "expected invalid but it is valid" << std::endl; + assert(false); + } else if (!valid[k] && !invalid[(startI + k) % pubKeys.size()]) { + std::cout << "expected valid but it is invalid" << std::endl; + assert(false); + } + } + } +} + +static void BLSVerify_BatchedParallel(benchmark::State& state) +{ + BLSPublicKeyVector pubKeys; + BLSSecretKeyVector secKeys; + BLSSignatureVector sigs; + std::vector msgHashes; + std::vector invalid; + BuildTestVectors(1000, 10, pubKeys, secKeys, sigs, msgHashes, invalid); + + std::list>> futures; + + volatile bool cancel = false; + auto cancelCond = [&]() { + return cancel; + }; + + // Benchmark. + size_t i = 0; + while (state.KeepRunning()) { + if (futures.size() < 100) { + while (futures.size() < 10000) { + auto f = blsWorker.AsyncVerifySig(sigs[i], pubKeys[i], msgHashes[i], cancelCond); + futures.emplace_back(std::make_pair(i, std::move(f))); + i = (i + 1) % pubKeys.size(); + } + } + + auto fp = std::move(futures.front()); + futures.pop_front(); + + size_t j = fp.first; + bool valid = fp.second.get(); + + if (valid && invalid[j]) { + std::cout << "expected invalid but it is valid" << std::endl; + assert(false); + } else if (!valid && !invalid[j]) { + std::cout << "expected valid but it is invalid" << std::endl; + assert(false); + } + } + cancel = true; + while (blsWorker.IsAsyncVerifyInProgress()) { + MilliSleep(100); + } +} + +BENCHMARK(BLSPubKeyAggregate_Normal) +BENCHMARK(BLSSecKeyAggregate_Normal) +BENCHMARK(BLSSign_Normal) +BENCHMARK(BLSVerify_Normal) +BENCHMARK(BLSVerify_LargeBlock1000) +BENCHMARK(BLSVerify_LargeBlockSelfAggregated1000) +BENCHMARK(BLSVerify_LargeBlockSelfAggregated10000) +BENCHMARK(BLSVerify_LargeAggregatedBlock1000) +BENCHMARK(BLSVerify_LargeAggregatedBlock10000) +BENCHMARK(BLSVerify_LargeAggregatedBlock1000PreVerified) +BENCHMARK(BLSVerify_Batched) +BENCHMARK(BLSVerify_BatchedParallel) diff --git a/src/bench/bls_dkg.cpp b/src/bench/bls_dkg.cpp new file mode 100644 index 000000000000..156886b339cf --- /dev/null +++ b/src/bench/bls_dkg.cpp @@ -0,0 +1,184 @@ +// Copyright (c) 2018 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" +#include "random.h" +#include "bls/bls_worker.h" + +extern CBLSWorker blsWorker; + +struct Member { + CBLSId id; + + BLSVerificationVectorPtr vvec; + BLSSecretKeyVector skShares; +}; + +struct DKG +{ + std::vector members; + std::vector ids; + + std::vector receivedVvecs; + BLSSecretKeyVector receivedSkShares; + + BLSVerificationVectorPtr quorumVvec; + + DKG(int quorumSize) + { + members.reserve(quorumSize); + ids.reserve(quorumSize); + + for (int i = 0; i < quorumSize; i++) { + uint256 id; + WriteLE64(id.begin(), i + 1); + members.push_back({id, {}, {}}); + ids.emplace_back(id); + } + + for (int i = 0; i < quorumSize; i++) { + blsWorker.GenerateContributions(quorumSize / 2 + 1, ids, members[i].vvec, members[i].skShares); + } + + //printf("initialized quorum %d\n", quorumSize); + } + + void ReceiveVvecs() + { + receivedVvecs.clear(); + for (size_t i = 0; i < members.size(); i++) { + receivedVvecs.emplace_back(members[i].vvec); + } + quorumVvec = blsWorker.BuildQuorumVerificationVector(receivedVvecs); + } + + void ReceiveShares(size_t whoAmI) + { + receivedSkShares.clear(); + for (size_t i = 0; i < members.size(); i++) { + receivedSkShares.emplace_back(members[i].skShares[whoAmI]); + } + } + + void BuildQuorumVerificationVector(bool parallel) + { + quorumVvec = blsWorker.BuildQuorumVerificationVector(receivedVvecs, 0, 0, parallel); + //assert(worker.VerifyVerificationVector(*members[memberIdx].quorumVvec)); + } + + void Bench_BuildQuorumVerificationVectors(benchmark::State& state, bool parallel) + { + ReceiveVvecs(); + + while (state.KeepRunning()) { + BuildQuorumVerificationVector(parallel); + } + } + + void VerifyContributionShares(size_t whoAmI, const std::set& invalidIndexes, bool parallel, bool aggregated) + { + auto result = blsWorker.VerifyContributionShares(members[whoAmI].id, receivedVvecs, receivedSkShares, parallel, aggregated); + for (size_t i = 0; i < receivedVvecs.size(); i++) { + if (invalidIndexes.count(i)) { + assert(!result[i]); + } else { + assert(result[i]); + } + } + } + + void Bench_VerifyContributionShares(benchmark::State& state, int invalidCount, bool parallel, bool aggregated) + { + ReceiveVvecs(); + + // Benchmark. + size_t memberIdx = 0; + while (state.KeepRunning()) { + auto& m = members[memberIdx]; + + ReceiveShares(memberIdx); + + std::set invalidIndexes; + for (int i = 0; i < invalidCount; i++) { + int shareIdx = GetRandInt(receivedSkShares.size()); + receivedSkShares[shareIdx].MakeNewKey(); + invalidIndexes.emplace(shareIdx); + } + + VerifyContributionShares(memberIdx, invalidIndexes, parallel, aggregated); + + memberIdx = (memberIdx + 1) % members.size(); + } + } +}; + +std::shared_ptr dkg10; +std::shared_ptr dkg100; +std::shared_ptr dkg400; + +void InitIfNeeded() +{ + if (dkg10 == nullptr) { + dkg10 = std::make_shared(10); + } + if (dkg100 == nullptr) { + dkg100 = std::make_shared(100); + } + if (dkg400 == nullptr) { + dkg400 = std::make_shared(400); + } +} + +void CleanupBLSDkgTests() +{ + dkg10.reset(); + dkg100.reset(); + dkg400.reset(); +} + + + +#define BENCH_BuildQuorumVerificationVectors(name, quorumSize, parallel) \ + static void BLSDKG_BuildQuorumVerificationVectors_##name##_##quorumSize(benchmark::State& state) \ + { \ + InitIfNeeded(); \ + dkg##quorumSize->Bench_BuildQuorumVerificationVectors(state, parallel); \ + } \ + BENCHMARK(BLSDKG_BuildQuorumVerificationVectors_##name##_##quorumSize) + +BENCH_BuildQuorumVerificationVectors(simple, 10, false) +BENCH_BuildQuorumVerificationVectors(simple, 100, false) +BENCH_BuildQuorumVerificationVectors(simple, 400, false) +BENCH_BuildQuorumVerificationVectors(parallel, 10, true) +BENCH_BuildQuorumVerificationVectors(parallel, 100, true) +BENCH_BuildQuorumVerificationVectors(parallel, 400, true) + +/////////////////////////////// + + + +#define BENCH_VerifyContributionShares(name, quorumSize, invalidCount, parallel, aggregated) \ + static void BLSDKG_VerifyContributionShares_##name##_##quorumSize(benchmark::State& state) \ + { \ + InitIfNeeded(); \ + dkg##quorumSize->Bench_VerifyContributionShares(state, invalidCount, parallel, aggregated); \ + } \ + BENCHMARK(BLSDKG_VerifyContributionShares_##name##_##quorumSize) + +BENCH_VerifyContributionShares(simple, 10, 5, false, false) +BENCH_VerifyContributionShares(simple, 100, 5, false, false) +BENCH_VerifyContributionShares(simple, 400, 5, false, false) + +BENCH_VerifyContributionShares(aggregated, 10, 5, false, true) +BENCH_VerifyContributionShares(aggregated, 100, 5, false, true) +BENCH_VerifyContributionShares(aggregated, 400, 5, false, true) + +BENCH_VerifyContributionShares(parallel, 10, 5, true, false) +BENCH_VerifyContributionShares(parallel, 100, 5, true, false) +BENCH_VerifyContributionShares(parallel, 400, 5, true, false) + +BENCH_VerifyContributionShares(parallel_aggregated, 10, 5, true, true) +BENCH_VerifyContributionShares(parallel_aggregated, 100, 5, true, true) +BENCH_VerifyContributionShares(parallel_aggregated, 400, 5, true, true) diff --git a/src/bench/ecdsa.cpp b/src/bench/ecdsa.cpp new file mode 100644 index 000000000000..706608eac208 --- /dev/null +++ b/src/bench/ecdsa.cpp @@ -0,0 +1,77 @@ +// Copyright (c) 2018 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bench.h" + +#include "key.h" + +static void ECDSASign(benchmark::State& state) +{ + std::vector keys; + std::vector hashes; + for (size_t i = 0; i < 100; i++) { + CKey k; + k.MakeNewKey(false); + keys.emplace_back(k); + hashes.emplace_back(::SerializeHash((int)i)); + } + + // Benchmark. + size_t i = 0; + while (state.KeepRunning()) { + std::vector sig; + keys[i].Sign(hashes[i], sig); + i = (i + 1) % keys.size(); + } +} + +static void ECDSAVerify(benchmark::State& state) +{ + std::vector keys; + std::vector hashes; + std::vector> sigs; + for (size_t i = 0; i < 100; i++) { + CKey k; + k.MakeNewKey(false); + keys.emplace_back(k.GetPubKey()); + hashes.emplace_back(::SerializeHash((int)i)); + std::vector sig; + k.Sign(hashes[i], sig); + sigs.emplace_back(sig); + } + + // Benchmark. + size_t i = 0; + while (state.KeepRunning()) { + keys[i].Verify(hashes[i], sigs[i]); + i = (i + 1) % keys.size(); + } +} + +static void ECDSAVerify_LargeBlock(benchmark::State& state) +{ + std::vector keys; + std::vector hashes; + std::vector> sigs; + for (size_t i = 0; i < 1000; i++) { + CKey k; + k.MakeNewKey(false); + keys.emplace_back(k.GetPubKey()); + hashes.emplace_back(::SerializeHash((int)i)); + std::vector sig; + k.Sign(hashes[i], sig); + sigs.emplace_back(sig); + } + + // Benchmark. + while (state.KeepRunning()) { + for (size_t i = 0; i < keys.size(); i++) { + keys[i].Verify(hashes[i], sigs[i]); + } + } +} + +BENCHMARK(ECDSASign) +BENCHMARK(ECDSAVerify) +BENCHMARK(ECDSAVerify_LargeBlock) diff --git a/src/bls/bls.cpp b/src/bls/bls.cpp new file mode 100644 index 000000000000..6105a0a15458 --- /dev/null +++ b/src/bls/bls.cpp @@ -0,0 +1,394 @@ +// Copyright (c) 2018 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bls.h" + +#include "hash.h" +#include "random.h" +#include "tinyformat.h" + +#ifndef BUILD_BITCOIN_INTERNAL +#include "support/allocators/mt_pooled_secure.h" +#endif + +#include +#include + +static std::unique_ptr pScheme(new bls::BasicSchemeMPL); + +CBLSId::CBLSId(const uint256& nHash) : CBLSWrapper() +{ + impl = nHash; + fValid = true; + cachedHash.SetNull(); +} + +void CBLSSecretKey::AggregateInsecure(const CBLSSecretKey& o) +{ + assert(IsValid() && o.IsValid()); + impl = bls::PrivateKey::Aggregate({impl, o.impl}); + cachedHash.SetNull(); +} + +CBLSSecretKey CBLSSecretKey::AggregateInsecure(const std::vector& sks) +{ + if (sks.empty()) { + return CBLSSecretKey(); + } + + std::vector v; + v.reserve(sks.size()); + for (auto& sk : sks) { + v.emplace_back(sk.impl); + } + + CBLSSecretKey ret; + ret.impl = bls::PrivateKey::Aggregate(v); + ret.fValid = true; + ret.cachedHash.SetNull(); + return ret; +} + +#ifndef BUILD_BITCOIN_INTERNAL +void CBLSSecretKey::MakeNewKey() +{ + unsigned char buf[32]; + while (true) { + GetStrongRandBytes(buf, sizeof(buf)); + try { + impl = bls::PrivateKey::FromBytes(bls::Bytes((const uint8_t*)buf, SerSize)); + break; + } catch (...) { + } + } + fValid = true; + cachedHash.SetNull(); +} +#endif + +bool CBLSSecretKey::SecretKeyShare(const std::vector& msk, const CBLSId& _id) +{ + fValid = false; + cachedHash.SetNull(); + + if (!_id.IsValid()) { + return false; + } + + std::vector mskVec; + mskVec.reserve(msk.size()); + for (const CBLSSecretKey& sk : msk) { + if (!sk.IsValid()) { + return false; + } + mskVec.emplace_back(sk.impl); + } + + try { + impl = bls::Threshold::PrivateKeyShare(mskVec, bls::Bytes(_id.impl.begin(), _id.impl.size())); + } catch (...) { + return false; + } + + fValid = true; + cachedHash.SetNull(); + return true; +} + +CBLSPublicKey CBLSSecretKey::GetPublicKey() const +{ + if (!IsValid()) { + return CBLSPublicKey(); + } + + CBLSPublicKey pubKey; + pubKey.impl = impl.GetG1Element(); + pubKey.fValid = true; + pubKey.cachedHash.SetNull(); + return pubKey; +} + +CBLSSignature CBLSSecretKey::Sign(const uint256& hash) const +{ + if (!IsValid()) { + return CBLSSignature(); + } + + CBLSSignature sigRet; + sigRet.impl = pScheme->Sign(impl, bls::Bytes(hash.begin(), hash.size())); + + sigRet.fValid = true; + sigRet.cachedHash.SetNull(); + + return sigRet; +} + +void CBLSPublicKey::AggregateInsecure(const CBLSPublicKey& o) +{ + assert(IsValid() && o.IsValid()); + impl = pScheme->Aggregate({impl, o.impl}); + cachedHash.SetNull(); +} + +CBLSPublicKey CBLSPublicKey::AggregateInsecure(const std::vector& pks) +{ + if (pks.empty()) { + return CBLSPublicKey(); + } + + std::vector vecPublicKeys; + vecPublicKeys.reserve(pks.size()); + for (auto& pk : pks) { + vecPublicKeys.emplace_back(pk.impl); + } + + CBLSPublicKey ret; + ret.impl = pScheme->Aggregate(vecPublicKeys); + ret.fValid = true; + ret.cachedHash.SetNull(); + return ret; +} + +bool CBLSPublicKey::PublicKeyShare(const std::vector& mpk, const CBLSId& _id) +{ + fValid = false; + cachedHash.SetNull(); + + if (!_id.IsValid()) { + return false; + } + + std::vector mpkVec; + mpkVec.reserve(mpk.size()); + for (const CBLSPublicKey& pk : mpk) { + if (!pk.IsValid()) { + return false; + } + mpkVec.emplace_back(pk.impl); + } + + try { + impl = bls::Threshold::PublicKeyShare(mpkVec, bls::Bytes(_id.impl.begin(), _id.impl.size())); + } catch (...) { + return false; + } + + fValid = true; + cachedHash.SetNull(); + return true; +} + +bool CBLSPublicKey::DHKeyExchange(const CBLSSecretKey& sk, const CBLSPublicKey& pk) +{ + fValid = false; + cachedHash.SetNull(); + + if (!sk.IsValid() || !pk.IsValid()) { + return false; + } + impl = sk.impl * pk.impl; + fValid = true; + cachedHash.SetNull(); + return true; +} + +void CBLSSignature::AggregateInsecure(const CBLSSignature& o) +{ + assert(IsValid() && o.IsValid()); + impl = pScheme->Aggregate({impl, o.impl}); + cachedHash.SetNull(); +} + +CBLSSignature CBLSSignature::AggregateInsecure(const std::vector& sigs) +{ + if (sigs.empty()) { + return CBLSSignature(); + } + + std::vector v; + v.reserve(sigs.size()); + for (auto& pk : sigs) { + v.emplace_back(pk.impl); + } + + CBLSSignature ret; + ret.impl = pScheme->Aggregate(v); + ret.fValid = true; + ret.cachedHash.SetNull(); + return ret; +} + +CBLSSignature CBLSSignature::AggregateSecure(const std::vector& sigs, + const std::vector& pks, + const uint256& hash) +{ + if (sigs.size() != pks.size() || sigs.empty()) { + return CBLSSignature(); + } + + std::vector vecPublicKeys; + vecPublicKeys.reserve(pks.size()); + for (auto& pk : pks) { + vecPublicKeys.push_back(pk.impl); + } + + std::vector vecSignatures; + vecSignatures.reserve(pks.size()); + for (auto& sig : sigs) { + vecSignatures.push_back(sig.impl); + } + + CBLSSignature ret; + ret.impl = pScheme->AggregateSecure(vecPublicKeys, vecSignatures, bls::Bytes(hash.begin(), hash.size())); + ret.fValid = true; + ret.cachedHash.SetNull(); + return ret; +} + +void CBLSSignature::SubInsecure(const CBLSSignature& o) +{ + assert(IsValid() && o.IsValid()); + impl = impl + o.impl.Negate(); + cachedHash.SetNull(); +} + +bool CBLSSignature::VerifyInsecure(const CBLSPublicKey& pubKey, const uint256& hash) const +{ + if (!IsValid() || !pubKey.IsValid()) { + return false; + } + + try { + return pScheme->Verify(pubKey.impl, bls::Bytes(hash.begin(), hash.size()), impl); + } catch (...) { + return false; + } +} + +bool CBLSSignature::VerifyInsecureAggregated(const std::vector& pubKeys, const std::vector& hashes) const +{ + if (!IsValid()) { + return false; + } + assert(!pubKeys.empty() && !hashes.empty() && pubKeys.size() == hashes.size()); + + std::vector pubKeyVec; + std::vector hashes2; + hashes2.reserve(hashes.size()); + pubKeyVec.reserve(pubKeys.size()); + for (size_t i = 0; i < pubKeys.size(); i++) { + auto& p = pubKeys[i]; + if (!p.IsValid()) { + return false; + } + pubKeyVec.push_back(p.impl); + hashes2.emplace_back(hashes[i].begin(), hashes[i].size()); + } + + try { + return pScheme->AggregateVerify(pubKeyVec, hashes2, impl); + } catch (...) { + return false; + } +} + +bool CBLSSignature::VerifySecureAggregated(const std::vector& pks, const uint256& hash) const +{ + if (pks.empty()) { + return false; + } + + std::vector vecPublicKeys; + vecPublicKeys.reserve(pks.size()); + for (const auto& pk : pks) { + vecPublicKeys.push_back(pk.impl); + } + + return pScheme->VerifySecure(vecPublicKeys, impl, bls::Bytes(hash.begin(), hash.size())); +} + +bool CBLSSignature::Recover(const std::vector& sigs, const std::vector& ids) +{ + fValid = false; + cachedHash.SetNull(); + + if (sigs.empty() || ids.empty() || sigs.size() != ids.size()) { + return false; + } + + std::vector sigsVec; + std::vector idsVec; + sigsVec.reserve(sigs.size()); + idsVec.reserve(sigs.size()); + + for (size_t i = 0; i < sigs.size(); i++) { + if (!sigs[i].IsValid() || !ids[i].IsValid()) { + return false; + } + sigsVec.emplace_back(sigs[i].impl); + idsVec.emplace_back(ids[i].impl.begin(), ids[i].impl.size()); + } + + try { + impl = bls::Threshold::SignatureRecover(sigsVec, idsVec); + } catch (...) { + return false; + } + + fValid = true; + cachedHash.SetNull(); + return true; +} + +#ifndef BUILD_BITCOIN_INTERNAL + +static std::once_flag init_flag; +static mt_pooled_secure_allocator* secure_allocator_instance; +static void create_secure_allocator() +{ + // make sure LockedPoolManager is initialized first (ensures destruction order) + LockedPoolManager::Instance(); + + // static variable in function scope ensures it's initialized when first accessed + // and destroyed before LockedPoolManager + static mt_pooled_secure_allocator a(sizeof(bn_t) + sizeof(size_t)); + secure_allocator_instance = &a; +} + +static mt_pooled_secure_allocator& get_secure_allocator() +{ + std::call_once(init_flag, create_secure_allocator); + return *secure_allocator_instance; +} + +static void* secure_allocate(size_t n) +{ + uint8_t* ptr = get_secure_allocator().allocate(n + sizeof(size_t)); + *(size_t*)ptr = n; + return ptr + sizeof(size_t); +} + +static void secure_free(void* p) +{ + if (!p) { + return; + } + + uint8_t* ptr = (uint8_t*)p - sizeof(size_t); + size_t n = *(size_t*)ptr; + return get_secure_allocator().deallocate(ptr, n); +} +#endif + +bool BLSInit() +{ + if (!bls::BLS::Init()) { + return false; + } +#ifndef BUILD_BITCOIN_INTERNAL + bls::BLS::SetSecureAllocator(secure_allocate, secure_free); +#endif + return true; +} diff --git a/src/bls/bls.h b/src/bls/bls.h new file mode 100644 index 000000000000..8db42cb940cc --- /dev/null +++ b/src/bls/bls.h @@ -0,0 +1,434 @@ +// Copyright (c) 2018 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef PIVX_CRYPTO_BLS_H +#define PIVX_CRYPTO_BLS_H + +#include "hash.h" +#include "serialize.h" +#include "uint256.h" +#include "utilstrencodings.h" + +// bls-dash uses relic, which may define DEBUG and ERROR, which leads to many warnings in some build setups +#undef ERROR +#undef DEBUG +#include +#include +#include +#include +#include +#undef DOUBLE + +#include +#include +#include + +// reversed BLS12-381 +#define BLS_CURVE_ID_SIZE 32 +#define BLS_CURVE_SECKEY_SIZE 32 +#define BLS_CURVE_PUBKEY_SIZE 48 +#define BLS_CURVE_SIG_SIZE 96 + +class CBLSSignature; +class CBLSPublicKey; + +template +class CBLSWrapper +{ + friend class CBLSSecretKey; + friend class CBLSPublicKey; + friend class CBLSSignature; + +protected: + ImplType impl; + bool fValid{false}; + mutable uint256 cachedHash; + + inline constexpr size_t GetSerSize() const { return SerSize; } + +public: + static const size_t SerSize = _SerSize; + + CBLSWrapper() + { + } + CBLSWrapper(const std::vector& vecBytes) : CBLSWrapper() + { + SetByteVector(vecBytes); + } + + CBLSWrapper(const CBLSWrapper& ref) = default; + CBLSWrapper& operator=(const CBLSWrapper& ref) = default; + CBLSWrapper(CBLSWrapper&& ref) + { + std::swap(impl, ref.impl); + std::swap(fValid, ref.fValid); + std::swap(cachedHash, ref.cachedHash); + } + CBLSWrapper& operator=(CBLSWrapper&& ref) + { + std::swap(impl, ref.impl); + std::swap(fValid, ref.fValid); + std::swap(cachedHash, ref.cachedHash); + return *this; + } + + bool operator==(const C& r) const + { + return fValid == r.fValid && impl == r.impl; + } + bool operator!=(const C& r) const + { + return !((*this) == r); + } + + bool IsValid() const + { + return fValid; + } + + void Reset() + { + *((C*)this) = C(); + } + + void SetByteVector(const std::vector& vecBytes) + { + if (vecBytes.size() != SerSize) { + Reset(); + return; + } + if (std::all_of(vecBytes.begin(), vecBytes.end(), [](uint8_t c) { return c == 0; })) { + Reset(); + } else { + try { + impl = ImplType::FromBytes(bls::Bytes(vecBytes)); + fValid = true; + } catch (...) { + Reset(); + } + } + cachedHash.SetNull(); + } + + std::vector ToByteVector() const + { + if (!fValid) { + return std::vector(SerSize, 0); + } + return impl.Serialize(); + } + + const uint256& GetHash() const + { + if (cachedHash.IsNull()) { + cachedHash = ::SerializeHash(*this); + } + return cachedHash; + } + + bool SetHexStr(const std::string& str) + { + if (!IsHex(str)) { + Reset(); + return false; + } + auto b = ParseHex(str); + if (b.size() != SerSize) { + Reset(); + return false; + } + SetByteVector(b); + return IsValid(); + } + +public: + template + inline void Serialize(Stream& s) const + { + s.write((const char*)ToByteVector().data(), SerSize); + } + + template + inline void Unserialize(Stream& s, bool checkMalleable = true) + { + std::vector vecBytes(SerSize, 0); + s.read((char*)vecBytes.data(), SerSize); + SetByteVector(vecBytes); + + if (checkMalleable && !CheckMalleable(vecBytes)) { + throw std::ios_base::failure("malleable BLS object"); + } + } + + inline bool CheckMalleable(const std::vector& vecBytes) const + { + if (memcmp(vecBytes.data(), ToByteVector().data(), SerSize)) { + // TODO not sure if this is actually possible with the BLS libs. I'm assuming here that somewhere deep inside + // these libs masking might happen, so that 2 different binary representations could result in the same object + // representation + return false; + } + return true; + } + + inline std::string ToString() const + { + std::vector buf = ToByteVector(); + return HexStr(buf.begin(), buf.end()); + } +}; + +struct CBLSIdImplicit : public uint256 +{ + CBLSIdImplicit() {} + CBLSIdImplicit(const uint256& id) + { + memcpy(begin(), id.begin(), sizeof(uint256)); + } + static CBLSIdImplicit FromBytes(const uint8_t* buffer) + { + CBLSIdImplicit instance; + memcpy(instance.begin(), buffer, sizeof(CBLSIdImplicit)); + return instance; + } + std::vector Serialize() const + { + return {begin(), end()}; + } +}; + +class CBLSId : public CBLSWrapper +{ +public: + using CBLSWrapper::operator=; + using CBLSWrapper::operator==; + using CBLSWrapper::operator!=; + + CBLSId() {} + CBLSId(const uint256& nHash); +}; + +class CBLSSecretKey : public CBLSWrapper +{ +public: + using CBLSWrapper::operator=; + using CBLSWrapper::operator==; + using CBLSWrapper::operator!=; + + void AggregateInsecure(const CBLSSecretKey& o); + static CBLSSecretKey AggregateInsecure(const std::vector& sks); + +#ifndef BUILD_BITCOIN_INTERNAL + void MakeNewKey(); +#endif + bool SecretKeyShare(const std::vector& msk, const CBLSId& id); + + CBLSPublicKey GetPublicKey() const; + CBLSSignature Sign(const uint256& hash) const; +}; + +class CBLSPublicKey : public CBLSWrapper +{ + friend class CBLSSecretKey; + friend class CBLSSignature; + +public: + using CBLSWrapper::operator=; + using CBLSWrapper::operator==; + using CBLSWrapper::operator!=; + using CBLSWrapper::CBLSWrapper; + + CBLSPublicKey() {} + + void AggregateInsecure(const CBLSPublicKey& o); + static CBLSPublicKey AggregateInsecure(const std::vector& pks); + + bool PublicKeyShare(const std::vector& mpk, const CBLSId& id); + bool DHKeyExchange(const CBLSSecretKey& sk, const CBLSPublicKey& pk); + +}; + +class CBLSSignature : public CBLSWrapper +{ + friend class CBLSSecretKey; + +public: + using CBLSWrapper::operator==; + using CBLSWrapper::operator!=; + using CBLSWrapper::CBLSWrapper; + + CBLSSignature() {} + CBLSSignature(const CBLSSignature&) = default; + CBLSSignature& operator=(const CBLSSignature&) = default; + + void AggregateInsecure(const CBLSSignature& o); + static CBLSSignature AggregateInsecure(const std::vector& sigs); + static CBLSSignature AggregateSecure(const std::vector& sigs, const std::vector& pks, const uint256& hash); + + void SubInsecure(const CBLSSignature& o); + + bool VerifyInsecure(const CBLSPublicKey& pubKey, const uint256& hash) const; + bool VerifyInsecureAggregated(const std::vector& pubKeys, const std::vector& hashes) const; + + bool VerifySecureAggregated(const std::vector& pks, const uint256& hash) const; + + bool Recover(const std::vector& sigs, const std::vector& ids); +}; + +#ifndef BUILD_BITCOIN_INTERNAL + +template +class CBLSLazyWrapper +{ +private: + mutable std::mutex mutex; + + mutable std::vector vecBytes; + mutable bool bufValid{false}; + + mutable BLSObject obj; + mutable bool objInitialized{false}; + + mutable uint256 hash; + +public: + CBLSLazyWrapper() : vecBytes(BLSObject::SerSize, 0) + { + // the all-zero buf is considered a valid buf, but the resulting object will return false for IsValid + bufValid = true; + } + + CBLSLazyWrapper(const CBLSLazyWrapper& r) + { + *this = r; + } + + CBLSLazyWrapper& operator=(const CBLSLazyWrapper& r) + { + std::unique_lock l(r.mutex); + bufValid = r.bufValid; + if (r.bufValid) { + vecBytes = r.vecBytes; + } else { + std::fill(vecBytes.begin(), vecBytes.end(), 0); + } + objInitialized = r.objInitialized; + if (r.objInitialized) { + obj = r.obj; + } else { + obj.Reset(); + } + hash = r.hash; + return *this; + } + + template + inline void Serialize(Stream& s) const + { + std::unique_lock l(mutex); + if (!objInitialized && !bufValid) { + throw std::ios_base::failure("obj and buf not initialized"); + } + if (!bufValid) { + vecBytes = obj.ToByteVector(); + bufValid = true; + hash.SetNull(); + } + s.write((const char*)vecBytes.data(), vecBytes.size()); + } + + template + inline void Unserialize(Stream& s) + { + std::unique_lock l(mutex); + s.read((char*)vecBytes.data(), BLSObject::SerSize); + bufValid = true; + objInitialized = false; + hash.SetNull(); + } + + void Set(const BLSObject& _obj) + { + std::unique_lock l(mutex); + bufValid = false; + objInitialized = true; + obj = _obj; + hash.SetNull(); + } + + const BLSObject& Get() const + { + std::unique_lock l(mutex); + static BLSObject invalidObj; + if (!bufValid && !objInitialized) { + return invalidObj; + } + if (!objInitialized) { + obj.SetByteVector(vecBytes); + if (!obj.CheckMalleable(vecBytes)) { + bufValid = false; + objInitialized = false; + obj = invalidObj; + } else { + objInitialized = true; + } + } + return obj; + } + + bool operator==(const CBLSLazyWrapper& r) const + { + if (bufValid && r.bufValid) { + return vecBytes == r.vecBytes; + } + if (objInitialized && r.objInitialized) { + return obj == r.obj; + } + return Get() == r.Get(); + } + + bool operator!=(const CBLSLazyWrapper& r) const + { + return !(*this == r); + } + + uint256 GetHash() const + { + std::unique_lock l(mutex); + if (!bufValid) { + vecBytes = obj.ToByteVector(); + bufValid = true; + hash.SetNull(); + } + if (hash.IsNull()) { + CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); + ss.write((const char*)vecBytes.data(), vecBytes.size()); + hash = ss.GetHash(); + } + return hash; + } +}; +typedef CBLSLazyWrapper CBLSLazySignature; +typedef CBLSLazyWrapper CBLSLazyPublicKey; +typedef CBLSLazyWrapper CBLSLazySecretKey; + +#endif + +typedef std::vector BLSIdVector; +typedef std::vector BLSVerificationVector; +typedef std::vector BLSPublicKeyVector; +typedef std::vector BLSSecretKeyVector; +typedef std::vector BLSSignatureVector; + +typedef std::shared_ptr BLSIdVectorPtr; +typedef std::shared_ptr BLSVerificationVectorPtr; +typedef std::shared_ptr BLSPublicKeyVectorPtr; +typedef std::shared_ptr BLSSecretKeyVectorPtr; +typedef std::shared_ptr BLSSignatureVectorPtr; + +bool BLSInit(); + +#endif // PIVX_CRYPTO_BLS_H diff --git a/src/bls/bls_ies.cpp b/src/bls/bls_ies.cpp new file mode 100644 index 000000000000..db0b8a56a855 --- /dev/null +++ b/src/bls/bls_ies.cpp @@ -0,0 +1,132 @@ +// Copyright (c) 2018 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bls_ies.h" + +#include "hash.h" +#include "random.h" +#include "streams.h" + +#include "crypto/aes.h" + +template +static bool EncryptBlob(const void* in, size_t inSize, Out& out, const void* symKey, const void* iv) +{ + out.resize(inSize); + + AES256CBCEncrypt enc((const unsigned char*)symKey, (const unsigned char*)iv, false); + int w = enc.Encrypt((const unsigned char*)in, (int)inSize, (unsigned char*)out.data()); + return w == (int)inSize; +} + +template +static bool DecryptBlob(const void* in, size_t inSize, Out& out, const void* symKey, const void* iv) +{ + out.resize(inSize); + + AES256CBCDecrypt enc((const unsigned char*)symKey, (const unsigned char*)iv, false); + int w = enc.Decrypt((const unsigned char*)in, (int)inSize, (unsigned char*)out.data()); + return w == (int)inSize; +} + +bool CBLSIESEncryptedBlob::Encrypt(const CBLSPublicKey& peerPubKey, const void* plainTextData, size_t dataSize) +{ + CBLSSecretKey ephemeralSecretKey; + ephemeralSecretKey.MakeNewKey(); + ephemeralPubKey = ephemeralSecretKey.GetPublicKey(); + GetStrongRandBytes(iv, sizeof(iv)); + + CBLSPublicKey pk; + if (!pk.DHKeyExchange(ephemeralSecretKey, peerPubKey)) { + return false; + } + + std::vector symKey = pk.ToByteVector(); + symKey.resize(32); + + return EncryptBlob(plainTextData, dataSize, data, symKey.data(), iv); +} + +bool CBLSIESEncryptedBlob::Decrypt(const CBLSSecretKey& secretKey, CDataStream& decryptedDataRet) const +{ + CBLSPublicKey pk; + if (!pk.DHKeyExchange(secretKey, ephemeralPubKey)) { + return false; + } + + std::vector symKey = pk.ToByteVector(); + symKey.resize(32); + + return DecryptBlob(data.data(), data.size(), decryptedDataRet, symKey.data(), iv); +} + + +bool CBLSIESMultiRecipientBlobs::Encrypt(const std::vector& recipients, const BlobVector& _blobs) +{ + if (recipients.size() != _blobs.size()) { + return false; + } + + InitEncrypt(_blobs.size()); + + for (size_t i = 0; i < _blobs.size(); i++) { + if (!Encrypt(i, recipients[i], _blobs[i])) { + return false; + } + } + + return true; +} + +void CBLSIESMultiRecipientBlobs::InitEncrypt(size_t count) +{ + ephemeralSecretKey.MakeNewKey(); + ephemeralPubKey = ephemeralSecretKey.GetPublicKey(); + GetStrongRandBytes(ivSeed.begin(), ivSeed.size()); + + uint256 iv = ivSeed; + ivVector.resize(count); + blobs.resize(count); + for (size_t i = 0; i < count; i++) { + ivVector[i] = iv; + iv = ::SerializeHash(iv); + } +} + +bool CBLSIESMultiRecipientBlobs::Encrypt(size_t idx, const CBLSPublicKey& recipient, const Blob& blob) +{ + assert(idx < blobs.size()); + + CBLSPublicKey pk; + if (!pk.DHKeyExchange(ephemeralSecretKey, recipient)) { + return false; + } + + std::vector symKey = pk.ToByteVector(); + symKey.resize(32); + + return EncryptBlob(blob.data(), blob.size(), blobs[idx], symKey.data(), ivVector[idx].begin()); +} + +bool CBLSIESMultiRecipientBlobs::Decrypt(size_t idx, const CBLSSecretKey& sk, Blob& blobRet) const +{ + if (idx >= blobs.size()) { + return false; + } + + CBLSPublicKey pk; + if (!pk.DHKeyExchange(sk, ephemeralPubKey)) { + return false; + } + + std::vector symKey = pk.ToByteVector(); + symKey.resize(32); + + uint256 iv = ivSeed; + for (size_t i = 0; i < idx; i++) { + iv = ::SerializeHash(iv); + } + + return DecryptBlob(blobs[idx].data(), blobs[idx].size(), blobRet, symKey.data(), iv.begin()); +} diff --git a/src/bls/bls_ies.h b/src/bls/bls_ies.h new file mode 100644 index 000000000000..ff4d4651ca49 --- /dev/null +++ b/src/bls/bls_ies.h @@ -0,0 +1,165 @@ +// Copyright (c) 2018 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef PIVX_CRYPTO_BLS_IES_H +#define PIVX_CRYPTO_BLS_IES_H + +#include "bls.h" +#include "streams.h" + +class CBLSIESEncryptedBlob +{ +public: + CBLSPublicKey ephemeralPubKey; + unsigned char iv[16]; + std::vector data; + + bool valid{false}; + +public: + ADD_SERIALIZE_METHODS + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + if (!ser_action.ForRead()) { + assert(valid); + } else { + valid = false; + } + READWRITE(ephemeralPubKey); + READWRITE(FLATDATA(iv)); + READWRITE(data); + if (ser_action.ForRead()) { + valid = true; + } + }; + +public: + bool Encrypt(const CBLSPublicKey& peerPubKey, const void* data, size_t dataSize); + bool Decrypt(const CBLSSecretKey& secretKey, CDataStream& decryptedDataRet) const; +}; + +template +class CBLSIESEncryptedObject : public CBLSIESEncryptedBlob +{ +public: + CBLSIESEncryptedObject() + { + } + + bool Encrypt(const CBLSPublicKey& peerPubKey, const Object& obj, int nVersion) + { + try { + CDataStream ds(SER_NETWORK, nVersion); + ds << obj; + return CBLSIESEncryptedBlob::Encrypt(peerPubKey, ds.data(), ds.size()); + } catch (std::exception&) { + return false; + } + } + + bool Decrypt(const CBLSSecretKey& secretKey, Object& objRet, int nVersion) const + { + CDataStream ds(SER_NETWORK, nVersion); + if (!CBLSIESEncryptedBlob::Decrypt(secretKey, ds)) { + return false; + } + try { + ds >> objRet; + } catch (std::exception& e) { + return false; + } + return true; + } +}; + +class CBLSIESMultiRecipientBlobs +{ +public: + typedef std::vector Blob; + typedef std::vector BlobVector; + +public: + CBLSPublicKey ephemeralPubKey; + uint256 ivSeed; + BlobVector blobs; + + // Used while encrypting. Temporary and only in-memory + CBLSSecretKey ephemeralSecretKey; + std::vector ivVector; + +public: + bool Encrypt(const std::vector& recipients, const BlobVector& _blobs); + + void InitEncrypt(size_t count); + bool Encrypt(size_t idx, const CBLSPublicKey& recipient, const Blob& blob); + bool Decrypt(size_t idx, const CBLSSecretKey& sk, Blob& blobRet) const; + +public: + ADD_SERIALIZE_METHODS + + template + inline void SerializationOp(Stream& s, Operation ser_action) + { + READWRITE(ephemeralPubKey); + READWRITE(ivSeed); + READWRITE(blobs); + } +}; + +template +class CBLSIESMultiRecipientObjects : public CBLSIESMultiRecipientBlobs +{ +public: + typedef std::vector ObjectVector; + +public: + bool Encrypt(const std::vector& recipients, const ObjectVector& _objects, int nVersion) + { + BlobVector blobs; + blobs.resize(_objects.size()); + + try { + CDataStream ds(SER_NETWORK, nVersion); + for (size_t i = 0; i < _objects.size(); i++) { + ds.clear(); + + ds << _objects[i]; + blobs[i].assign(ds.begin(), ds.end()); + } + } catch (std::exception&) { + return false; + } + + return CBLSIESMultiRecipientBlobs::Encrypt(recipients, blobs); + } + + bool Encrypt(size_t idx, const CBLSPublicKey& recipient, const Object& obj, int nVersion) + { + CDataStream ds(SER_NETWORK, nVersion); + ds << obj; + Blob blob(ds.begin(), ds.end()); + return CBLSIESMultiRecipientBlobs::Encrypt(idx, recipient, blob); + } + + bool Decrypt(size_t idx, const CBLSSecretKey& sk, Object& objectRet, int nVersion) const + { + Blob blob; + if (!CBLSIESMultiRecipientBlobs::Decrypt(idx, sk, blob)) { + return false; + } + + try { + CDataStream ds(blob, SER_NETWORK, nVersion); + ds >> objectRet; + return true; + } catch (std::exception&) { + return false; + } + } +}; + +#endif // PIVX_CRYPTO_BLS_IES_H diff --git a/src/bls/bls_worker.cpp b/src/bls/bls_worker.cpp new file mode 100644 index 000000000000..231b9bda0af6 --- /dev/null +++ b/src/bls/bls_worker.cpp @@ -0,0 +1,962 @@ +// Copyright (c) 2018 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "bls_worker.h" +#include "hash.h" +#include "serialize.h" +#include "util/threadnames.h" + + +template +bool VerifyVectorHelper(const std::vector& vec, size_t start, size_t count) +{ + if (start == 0 && count == 0) { + count = vec.size(); + } + std::set set; + for (size_t i = start; i < start + count; i++) { + if (!vec[i].IsValid()) + return false; + // check duplicates + if (!set.emplace(vec[i].GetHash()).second) { + return false; + } + } + return true; +} + +// Creates a doneCallback and a future. The doneCallback simply finishes the future +template +std::pair, std::future > BuildFutureDoneCallback() +{ + auto p = std::make_shared >(); + std::function f = [p](const T& v) { + p->set_value(v); + }; + return std::make_pair(std::move(f), p->get_future()); +} +template +std::pair, std::future > BuildFutureDoneCallback2() +{ + auto p = std::make_shared >(); + std::function f = [p](T v) { + p->set_value(v); + }; + return std::make_pair(std::move(f), p->get_future()); +} + + +///// + +CBLSWorker::CBLSWorker() +{ +} + +CBLSWorker::~CBLSWorker() +{ + Stop(); +} + +void CBLSWorker::Start() +{ + int workerCount = std::thread::hardware_concurrency() / 2; + workerCount = std::max(std::min(1, workerCount), 4); + workerPool.resize(workerCount); + + RenameThreadPool(workerPool, "bls-worker"); +} + +void CBLSWorker::Stop() +{ + workerPool.clear_queue(); + workerPool.stop(true); +} + +bool CBLSWorker::GenerateContributions(int quorumThreshold, const BLSIdVector& ids, BLSVerificationVectorPtr& vvecRet, BLSSecretKeyVector& skShares) +{ + BLSSecretKeyVectorPtr svec = std::make_shared((size_t)quorumThreshold); + vvecRet = std::make_shared((size_t)quorumThreshold); + skShares.resize(ids.size()); + + for (int i = 0; i < quorumThreshold; i++) { + (*svec)[i].MakeNewKey(); + } + std::list > futures; + size_t batchSize = 8; + + for (size_t i = 0; i < quorumThreshold; i += batchSize) { + size_t start = i; + size_t count = std::min(batchSize, quorumThreshold - start); + auto f = [&, start, count](int threadId) { + for (size_t j = start; j < start + count; j++) { + (*vvecRet)[j] = (*svec)[j].GetPublicKey(); + } + return true; + }; + futures.emplace_back(workerPool.push(f)); + } + + for (size_t i = 0; i < ids.size(); i += batchSize) { + size_t start = i; + size_t count = std::min(batchSize, ids.size() - start); + auto f = [&, start, count](int threadId) { + for (size_t j = start; j < start + count; j++) { + if (!skShares[j].SecretKeyShare(*svec, ids[j])) { + return false; + } + } + return true; + }; + futures.emplace_back(workerPool.push(f)); + } + bool success = true; + for (auto& f : futures) { + if (!f.get()) { + success = false; + } + } + return success; +} + +// aggregates a single vector of BLS objects in parallel +// the input vector is split into batches and each batch is aggregated in parallel +// when enough batches are finished to form a new batch, the new batch is queued for further parallel aggregation +// when no more batches can be created from finished batch results, the final aggregated is created and the doneCallback +// called. +// The Aggregator object needs to be created on the heap and it will delete itself after calling the doneCallback +// The input vector is not copied into the Aggregator but instead a vector of pointers to the original entries from the +// input vector is stored. This means that the input vector must stay alive for the whole lifetime of the Aggregator +template +struct Aggregator { + typedef T ElementType; + + size_t batchSize{16}; + std::shared_ptr > inputVec; + + bool parallel; + ctpl::thread_pool& workerPool; + + std::mutex m; + // items in the queue are all intermediate aggregation results of finished batches. + // The intermediate results must be deleted by us again (which we do in SyncAggregateAndPushAggQueue) + boost::lockfree::queue aggQueue; + std::atomic aggQueueSize{0}; + + // keeps track of currently queued/in-progress batches. If it reaches 0, we are done + std::atomic waitCount{0}; + + typedef std::function DoneCallback; + DoneCallback doneCallback; + + // TP can either be a pointer or a reference + template + Aggregator(const std::vector& _inputVec, + size_t start, size_t count, + bool _parallel, + ctpl::thread_pool& _workerPool, + DoneCallback _doneCallback) : + workerPool(_workerPool), + parallel(_parallel), + aggQueue(0), + doneCallback(std::move(_doneCallback)) + { + inputVec = std::make_shared >(count); + for (size_t i = 0; i < count; i++) { + (*inputVec)[i] = pointer(_inputVec[start + i]); + } + } + + const T* pointer(const T& v) { return &v; } + const T* pointer(const T* v) { return v; } + + // Starts aggregation. + // If parallel=true, then this will return fast, otherwise this will block until aggregation is done + void Start() + { + size_t batchCount = (inputVec->size() + batchSize - 1) / batchSize; + + if (!parallel) { + if (inputVec->size() == 1) { + doneCallback(*(*inputVec)[0]); + } else { + doneCallback(SyncAggregate(*inputVec, 0, inputVec->size())); + } + delete this; + return; + } + + if (batchCount == 1) { + // just a single batch of work, take a shortcut. + PushWork([this](int threadId) { + if (inputVec->size() == 1) { + doneCallback(*(*inputVec)[0]); + } else { + doneCallback(SyncAggregate(*inputVec, 0, inputVec->size())); + } + delete this; + }); + return; + } + + // increment wait counter as otherwise the first finished async aggregation might signal that we're done + IncWait(); + for (size_t i = 0; i < batchCount; i++) { + size_t start = i * batchSize; + size_t count = std::min(batchSize, inputVec->size() - start); + AsyncAggregateAndPushAggQueue(inputVec, start, count, false); + } + // this will decrement the wait counter and in most cases NOT finish, as async work is still in progress + CheckDone(); + } + + void IncWait() + { + ++waitCount; + } + + void CheckDone() + { + if (--waitCount == 0) { + Finish(); + } + } + + void Finish() + { + // All async work is done, but we might have items in the aggQueue which are the results of the async + // work. This is the case when these did not add up to a new batch. In this case, we have to aggregate + // the items into the final result + + std::vector rem(aggQueueSize); + for (size_t i = 0; i < rem.size(); i++) { + T* p = nullptr; + bool s = aggQueue.pop(p); + assert(s); + rem[i] = p; + } + + T r; + if (rem.size() == 1) { + // just one intermediate result, which is actually the final result + r = *rem[0]; + } else { + // multiple intermediate results left which did not add up to a new batch. aggregate them now + r = SyncAggregate(rem, 0, rem.size()); + } + + // all items which are left in the queue are intermediate results, so we must delete them + for (size_t i = 0; i < rem.size(); i++) { + delete rem[i]; + } + doneCallback(r); + + delete this; + } + + void AsyncAggregateAndPushAggQueue(std::shared_ptr >& vec, size_t start, size_t count, bool del) + { + IncWait(); + PushWork(std::bind(&Aggregator::SyncAggregateAndPushAggQueue, this, vec, start, count, del)); + } + + void SyncAggregateAndPushAggQueue(std::shared_ptr >& vec, size_t start, size_t count, bool del) + { + // aggregate vec and push the intermediate result onto the work queue + PushAggQueue(SyncAggregate(*vec, start, count)); + if (del) { + for (size_t i = 0; i < count; i++) { + delete (*vec)[start + i]; + } + } + CheckDone(); + } + + void PushAggQueue(const T& v) + { + aggQueue.push(new T(v)); + + if (++aggQueueSize >= batchSize) { + // we've collected enough intermediate results to form a new batch. + std::shared_ptr > newBatch; + { + std::unique_lock l(m); + if (aggQueueSize < batchSize) { + // some other worker thread grabbed this batch + return; + } + newBatch = std::make_shared >(batchSize); + // collect items for new batch + for (size_t i = 0; i < batchSize; i++) { + T* p = nullptr; + bool s = aggQueue.pop(p); + assert(s); + (*newBatch)[i] = p; + } + aggQueueSize -= batchSize; + } + + // push new batch to work queue. del=true this time as these items are intermediate results and need to be deleted + // after aggregation is done + AsyncAggregateAndPushAggQueue(newBatch, 0, newBatch->size(), true); + } + } + + template + T SyncAggregate(const std::vector& vec, size_t start, size_t count) + { + T result = *vec[start]; + for (size_t j = 1; j < count; j++) { + result.AggregateInsecure(*vec[start + j]); + } + return result; + } + + template + void PushWork(Callable&& f) + { + workerPool.push(f); + } +}; + +// Aggregates multiple input vectors into a single output vector +// Inputs are in the following form: +// [ +// [a1, b1, c1, d1], +// [a2, b2, c2, d2], +// [a3, b3, c3, d3], +// [a4, b4, c4, d4], +// ] +// The result is in the following form: +// [ a1+a2+a3+a4, b1+b2+b3+b4, c1+c2+c3+c4, d1+d2+d3+d4] +// Same rules for the input vectors apply to the VectorAggregator as for the Aggregator (they must stay alive) +template +struct VectorAggregator { + typedef Aggregator AggregatorType; + typedef std::vector VectorType; + typedef std::shared_ptr VectorPtrType; + typedef std::vector VectorVectorType; + typedef std::function DoneCallback; + DoneCallback doneCallback; + + const VectorVectorType& vecs; + size_t start; + size_t count; + bool parallel; + ctpl::thread_pool& workerPool; + + std::atomic doneCount; + + VectorPtrType result; + size_t vecSize; + + VectorAggregator(const VectorVectorType& _vecs, + size_t _start, size_t _count, + bool _parallel, ctpl::thread_pool& _workerPool, + DoneCallback _doneCallback) : + vecs(_vecs), + parallel(_parallel), + start(_start), + count(_count), + workerPool(_workerPool), + doneCallback(std::move(_doneCallback)) + { + assert(!vecs.empty()); + vecSize = vecs[0]->size(); + result = std::make_shared(vecSize); + doneCount = 0; + } + + void Start() + { + std::vector aggregators; + for (size_t i = 0; i < vecSize; i++) { + std::vector tmp(count); + for (size_t j = 0; j < count; j++) { + tmp[j] = &(*vecs[start + j])[i]; + } + + auto aggregator = new AggregatorType(std::move(tmp), 0, count, parallel, workerPool, std::bind(&VectorAggregator::CheckDone, this, std::placeholders::_1, i)); + // we can't directly start the aggregator here as it might be so fast that it deletes "this" while we are still in this loop + aggregators.emplace_back(aggregator); + } + for (auto agg : aggregators) { + agg->Start(); + } + } + + void CheckDone(const T& agg, size_t idx) + { + (*result)[idx] = agg; + if (++doneCount == vecSize) { + doneCallback(result); + delete this; + } + } +}; + +// See comment of AsyncVerifyContributionShares for a description on what this does +// Same rules as in Aggregator apply for the inputs +struct ContributionVerifier { + struct BatchState { + size_t start; + size_t count; + + BLSVerificationVectorPtr vvec; + CBLSSecretKey skShare; + + // starts with 0 and is incremented if either vvec or skShare aggregation finishs. If it reaches 2, we know + // that aggregation for this batch is fully done. We can then start verification. + std::unique_ptr > aggDone; + + // we can't directly update a vector in parallel + // as vector is not thread safe (uses bitsets internally) + // so we must use vector temporarely and concatenate/convert + // each batch result into a final vector + std::vector verifyResults; + }; + + CBLSId forId; + const std::vector& vvecs; + const BLSSecretKeyVector& skShares; + size_t batchSize; + bool parallel; + bool aggregated; + + ctpl::thread_pool& workerPool; + + size_t batchCount; + size_t verifyCount; + + std::vector batchStates; + std::atomic verifyDoneCount{0}; + std::function&)> doneCallback; + + ContributionVerifier(const CBLSId& _forId, const std::vector& _vvecs, + const BLSSecretKeyVector& _skShares, size_t _batchSize, + bool _parallel, bool _aggregated, ctpl::thread_pool& _workerPool, + std::function&)> _doneCallback) : + forId(_forId), + vvecs(_vvecs), + skShares(_skShares), + batchSize(_batchSize), + parallel(_parallel), + aggregated(_aggregated), + workerPool(_workerPool), + doneCallback(std::move(_doneCallback)) + { + } + + void Start() + { + if (!aggregated) { + // treat all inputs as one large batch + batchSize = vvecs.size(); + batchCount = 1; + } else { + batchCount = (vvecs.size() + batchSize - 1) / batchSize; + } + verifyCount = vvecs.size(); + + batchStates.resize(batchCount); + for (size_t i = 0; i < batchCount; i++) { + auto& batchState = batchStates[i]; + + batchState.aggDone.reset(new std::atomic(0)); + batchState.start = i * batchSize; + batchState.count = std::min(batchSize, vvecs.size() - batchState.start); + batchState.verifyResults.assign(batchState.count, 0); + } + + if (aggregated) { + size_t batchCount2 = batchCount; // 'this' might get deleted while we're still looping + for (size_t i = 0; i < batchCount2; i++) { + AsyncAggregate(i); + } + } else { + // treat all inputs as a single batch and verify one-by-one + AsyncVerifyBatchOneByOne(0); + } + } + + void Finish() + { + size_t batchIdx = 0; + std::vector result(vvecs.size()); + for (size_t i = 0; i < vvecs.size(); i += batchSize) { + auto& batchState = batchStates[batchIdx++]; + for (size_t j = 0; j < batchState.count; j++) { + result[batchState.start + j] = batchState.verifyResults[j] != 0; + } + } + doneCallback(result); + delete this; + } + + void AsyncAggregate(size_t batchIdx) + { + auto& batchState = batchStates[batchIdx]; + + // aggregate vvecs and skShares of batch in parallel + auto vvecAgg = new VectorAggregator(vvecs, batchState.start, batchState.count, parallel, workerPool, std::bind(&ContributionVerifier::HandleAggVvecDone, this, batchIdx, std::placeholders::_1)); + auto skShareAgg = new Aggregator(skShares, batchState.start, batchState.count, parallel, workerPool, std::bind(&ContributionVerifier::HandleAggSkShareDone, this, batchIdx, std::placeholders::_1)); + + vvecAgg->Start(); + skShareAgg->Start(); + } + + void HandleAggVvecDone(size_t batchIdx, const BLSVerificationVectorPtr& vvec) + { + auto& batchState = batchStates[batchIdx]; + batchState.vvec = vvec; + if (++(*batchState.aggDone) == 2) { + HandleAggDone(batchIdx); + } + } + void HandleAggSkShareDone(size_t batchIdx, const CBLSSecretKey& skShare) + { + auto& batchState = batchStates[batchIdx]; + batchState.skShare = skShare; + if (++(*batchState.aggDone) == 2) { + HandleAggDone(batchIdx); + } + } + + void HandleVerifyDone(size_t batchIdx, size_t count) + { + size_t c = verifyDoneCount += count; + if (c == verifyCount) { + Finish(); + } + } + + void HandleAggDone(size_t batchIdx) + { + auto& batchState = batchStates[batchIdx]; + + if (batchState.vvec == nullptr || batchState.vvec->empty() || !batchState.skShare.IsValid()) { + // something went wrong while aggregating and there is nothing we can do now except mark the whole batch as failed + // this can only happen if inputs were invalid in some way + batchState.verifyResults.assign(batchState.count, 0); + HandleVerifyDone(batchIdx, batchState.count); + return; + } + + AsyncAggregatedVerifyBatch(batchIdx); + } + + void AsyncAggregatedVerifyBatch(size_t batchIdx) + { + auto f = [this, batchIdx](int threadId) { + auto& batchState = batchStates[batchIdx]; + bool result = Verify(batchState.vvec, batchState.skShare); + if (result) { + // whole batch is valid + batchState.verifyResults.assign(batchState.count, 1); + HandleVerifyDone(batchIdx, batchState.count); + } else { + // at least one entry in the batch is invalid, revert to per-contribution verification (but parallelized) + AsyncVerifyBatchOneByOne(batchIdx); + } + }; + PushOrDoWork(std::move(f)); + } + + void AsyncVerifyBatchOneByOne(size_t batchIdx) + { + size_t count = batchStates[batchIdx].count; + batchStates[batchIdx].verifyResults.assign(count, 0); + for (size_t i = 0; i < count; i++) { + auto f = [this, i, batchIdx](int threadId) { + auto& batchState = batchStates[batchIdx]; + batchState.verifyResults[i] = Verify(vvecs[batchState.start + i], skShares[batchState.start + i]); + HandleVerifyDone(batchIdx, 1); + }; + PushOrDoWork(std::move(f)); + } + } + + bool Verify(const BLSVerificationVectorPtr& vvec, const CBLSSecretKey& skShare) + { + CBLSPublicKey pk1; + if (!pk1.PublicKeyShare(*vvec, forId)) { + return false; + } + + CBLSPublicKey pk2 = skShare.GetPublicKey(); + return pk1 == pk2; + } + + template + void PushOrDoWork(Callable&& f) + { + if (parallel) { + workerPool.push(std::move(f)); + } else { + f(0); + } + } +}; + +void CBLSWorker::AsyncBuildQuorumVerificationVector(const std::vector& vvecs, + size_t start, size_t count, bool parallel, + std::function doneCallback) +{ + if (start == 0 && count == 0) { + count = vvecs.size(); + } + if (vvecs.empty() || count == 0 || start > vvecs.size() || start + count > vvecs.size()) { + doneCallback(nullptr); + return; + } + if (!VerifyVerificationVectors(vvecs, start, count)) { + doneCallback(nullptr); + return; + } + + auto agg = new VectorAggregator(vvecs, start, count, parallel, workerPool, std::move(doneCallback)); + agg->Start(); +} + +std::future CBLSWorker::AsyncBuildQuorumVerificationVector(const std::vector& vvecs, + size_t start, size_t count, bool parallel) +{ + auto p = BuildFutureDoneCallback(); + AsyncBuildQuorumVerificationVector(vvecs, start, count, parallel, std::move(p.first)); + return std::move(p.second); +} + +BLSVerificationVectorPtr CBLSWorker::BuildQuorumVerificationVector(const std::vector& vvecs, + size_t start, size_t count, bool parallel) +{ + return AsyncBuildQuorumVerificationVector(vvecs, start, count, parallel).get(); +} + +template +void AsyncAggregateHelper(ctpl::thread_pool& workerPool, + const std::vector& vec, size_t start, size_t count, bool parallel, + std::function doneCallback) +{ + if (start == 0 && count == 0) { + count = vec.size(); + } + if (vec.empty() || count == 0 || start > vec.size() || start + count > vec.size()) { + doneCallback(T()); + return; + } + if (!VerifyVectorHelper(vec, start, count)) { + doneCallback(T()); + return; + } + + auto agg = new Aggregator(vec, start, count, parallel, workerPool, std::move(doneCallback)); + agg->Start(); +} + +void CBLSWorker::AsyncAggregateSecretKeys(const BLSSecretKeyVector& secKeys, + size_t start, size_t count, bool parallel, + std::function doneCallback) +{ + AsyncAggregateHelper(workerPool, secKeys, start, count, parallel, doneCallback); +} + +std::future CBLSWorker::AsyncAggregateSecretKeys(const BLSSecretKeyVector& secKeys, + size_t start, size_t count, bool parallel) +{ + auto p = BuildFutureDoneCallback(); + AsyncAggregateSecretKeys(secKeys, start, count, parallel, std::move(p.first)); + return std::move(p.second); +} + +CBLSSecretKey CBLSWorker::AggregateSecretKeys(const BLSSecretKeyVector& secKeys, + size_t start, size_t count, bool parallel) +{ + return AsyncAggregateSecretKeys(secKeys, start, count, parallel).get(); +} + +void CBLSWorker::AsyncAggregatePublicKeys(const BLSPublicKeyVector& pubKeys, + size_t start, size_t count, bool parallel, + std::function doneCallback) +{ + AsyncAggregateHelper(workerPool, pubKeys, start, count, parallel, doneCallback); +} + +std::future CBLSWorker::AsyncAggregatePublicKeys(const BLSPublicKeyVector& pubKeys, + size_t start, size_t count, bool parallel) +{ + auto p = BuildFutureDoneCallback(); + AsyncAggregatePublicKeys(pubKeys, start, count, parallel, std::move(p.first)); + return std::move(p.second); +} + +CBLSPublicKey CBLSWorker::AggregatePublicKeys(const BLSPublicKeyVector& pubKeys, + size_t start, size_t count, bool parallel) +{ + return AsyncAggregatePublicKeys(pubKeys, start, count, parallel).get(); +} + +void CBLSWorker::AsyncAggregateSigs(const BLSSignatureVector& sigs, + size_t start, size_t count, bool parallel, + std::function doneCallback) +{ + AsyncAggregateHelper(workerPool, sigs, start, count, parallel, doneCallback); +} + +std::future CBLSWorker::AsyncAggregateSigs(const BLSSignatureVector& sigs, + size_t start, size_t count, bool parallel) +{ + auto p = BuildFutureDoneCallback(); + AsyncAggregateSigs(sigs, start, count, parallel, std::move(p.first)); + return std::move(p.second); +} + +CBLSSignature CBLSWorker::AggregateSigs(const BLSSignatureVector& sigs, + size_t start, size_t count, bool parallel) +{ + return AsyncAggregateSigs(sigs, start, count, parallel).get(); +} + + +CBLSPublicKey CBLSWorker::BuildPubKeyShare(const BLSVerificationVectorPtr& vvec, const CBLSId& id) +{ + CBLSPublicKey pkShare; + pkShare.PublicKeyShare(*vvec, id); + return pkShare; +} + +void CBLSWorker::AsyncVerifyContributionShares(const CBLSId& forId, const std::vector& vvecs, const BLSSecretKeyVector& skShares, + bool parallel, bool aggregated, std::function&)> doneCallback) +{ + if (!forId.IsValid() || !VerifyVerificationVectors(vvecs)) { + std::vector result; + result.assign(vvecs.size(), false); + doneCallback(result); + return; + } + + auto verifier = new ContributionVerifier(forId, vvecs, skShares, 8, parallel, aggregated, workerPool, std::move(doneCallback)); + verifier->Start(); +} + +std::future > CBLSWorker::AsyncVerifyContributionShares(const CBLSId& forId, const std::vector& vvecs, const BLSSecretKeyVector& skShares, + bool parallel, bool aggregated) +{ + auto p = BuildFutureDoneCallback >(); + AsyncVerifyContributionShares(forId, vvecs, skShares, parallel, aggregated, std::move(p.first)); + return std::move(p.second); +} + +std::vector CBLSWorker::VerifyContributionShares(const CBLSId& forId, const std::vector& vvecs, const BLSSecretKeyVector& skShares, + bool parallel, bool aggregated) +{ + return AsyncVerifyContributionShares(forId, vvecs, skShares, parallel, aggregated).get(); +} + +std::future CBLSWorker::AsyncVerifyContributionShare(const CBLSId& forId, + const BLSVerificationVectorPtr& vvec, + const CBLSSecretKey& skContribution) +{ + if (!forId.IsValid() || !VerifyVerificationVector(*vvec)) { + auto p = BuildFutureDoneCallback(); + p.first(false); + return std::move(p.second); + } + + auto f = [this, &forId, &vvec, &skContribution](int threadId) { + CBLSPublicKey pk1; + if (!pk1.PublicKeyShare(*vvec, forId)) { + return false; + } + + CBLSPublicKey pk2 = skContribution.GetPublicKey(); + return pk1 == pk2; + }; + return workerPool.push(f); +} + +bool CBLSWorker::VerifyContributionShare(const CBLSId& forId, const BLSVerificationVectorPtr& vvec, + const CBLSSecretKey& skContribution) +{ + CBLSPublicKey pk1; + if (!pk1.PublicKeyShare(*vvec, forId)) { + return false; + } + + CBLSPublicKey pk2 = skContribution.GetPublicKey(); + return pk1 == pk2; +} + +bool CBLSWorker::VerifyVerificationVector(const BLSVerificationVector& vvec, size_t start, size_t count) +{ + return VerifyVectorHelper(vvec, start, count); +} + +bool CBLSWorker::VerifyVerificationVectors(const std::vector& vvecs, + size_t start, size_t count) +{ + if (start == 0 && count == 0) { + count = vvecs.size(); + } + + std::set set; + for (size_t i = 0; i < count; i++) { + auto& vvec = vvecs[start + i]; + if (vvec == nullptr) { + return false; + } + if (vvec->size() != vvecs[start]->size()) { + return false; + } + for (size_t j = 0; j < vvec->size(); j++) { + if (!(*vvec)[j].IsValid()) { + return false; + } + // check duplicates + if (!set.emplace((*vvec)[j].GetHash()).second) { + return false; + } + } + } + + return true; +} + +bool CBLSWorker::VerifySecretKeyVector(const BLSSecretKeyVector& secKeys, size_t start, size_t count) +{ + return VerifyVectorHelper(secKeys, start, count); +} + +bool CBLSWorker::VerifySignatureVector(const BLSSignatureVector& sigs, size_t start, size_t count) +{ + return VerifyVectorHelper(sigs, start, count); +} + +void CBLSWorker::AsyncSign(const CBLSSecretKey& secKey, const uint256& msgHash, CBLSWorker::SignDoneCallback doneCallback) +{ + workerPool.push([secKey, msgHash, doneCallback](int threadId) { + doneCallback(secKey.Sign(msgHash)); + }); +} + +std::future CBLSWorker::AsyncSign(const CBLSSecretKey& secKey, const uint256& msgHash) +{ + auto p = BuildFutureDoneCallback(); + AsyncSign(secKey, msgHash, std::move(p.first)); + return std::move(p.second); +} + +void CBLSWorker::AsyncVerifySig(const CBLSSignature& sig, const CBLSPublicKey& pubKey, const uint256& msgHash, + CBLSWorker::SigVerifyDoneCallback doneCallback, CancelCond cancelCond) +{ + if (!sig.IsValid() || !pubKey.IsValid()) { + doneCallback(false); + return; + } + + std::unique_lock l(sigVerifyMutex); + + bool foundDuplicate = false; + for (auto& s : sigVerifyQueue) { + if (s.msgHash == msgHash) { + foundDuplicate = true; + break; + } + } + + if (foundDuplicate) { + // batched/aggregated verification does not allow duplicate hashes, so we push what we currently have and start + // with a fresh batch + PushSigVerifyBatch(); + } + + sigVerifyQueue.emplace_back(std::move(doneCallback), std::move(cancelCond), sig, pubKey, msgHash); + if (sigVerifyBatchesInProgress == 0 || sigVerifyQueue.size() >= SIG_VERIFY_BATCH_SIZE) { + PushSigVerifyBatch(); + } +} + +std::future CBLSWorker::AsyncVerifySig(const CBLSSignature& sig, const CBLSPublicKey& pubKey, const uint256& msgHash, CancelCond cancelCond) +{ + auto p = BuildFutureDoneCallback2(); + AsyncVerifySig(sig, pubKey, msgHash, std::move(p.first), cancelCond); + return std::move(p.second); +} + +bool CBLSWorker::IsAsyncVerifyInProgress() +{ + std::unique_lock l(sigVerifyMutex); + return sigVerifyBatchesInProgress != 0; +} + +// sigVerifyMutex must be held while calling +void CBLSWorker::PushSigVerifyBatch() +{ + auto f = [this](int threadId, std::shared_ptr > _jobs) { + auto& jobs = *_jobs; + if (jobs.size() == 1) { + auto& job = jobs[0]; + if (!job.cancelCond()) { + bool valid = job.sig.VerifyInsecure(job.pubKey, job.msgHash); + job.doneCallback(valid); + } + std::unique_lock l(sigVerifyMutex); + sigVerifyBatchesInProgress--; + if (!sigVerifyQueue.empty()) { + PushSigVerifyBatch(); + } + return; + } + + CBLSSignature aggSig; + std::vector indexes; + std::vector pubKeys; + std::vector msgHashes; + indexes.reserve(jobs.size()); + pubKeys.reserve(jobs.size()); + msgHashes.reserve(jobs.size()); + for (size_t i = 0; i < jobs.size(); i++) { + auto& job = jobs[i]; + if (job.cancelCond()) { + continue; + } + if (pubKeys.empty()) { + aggSig = job.sig; + } else { + aggSig.AggregateInsecure(job.sig); + } + indexes.emplace_back(i); + pubKeys.emplace_back(job.pubKey); + msgHashes.emplace_back(job.msgHash); + } + + if (!pubKeys.empty()) { + bool allValid = aggSig.VerifyInsecureAggregated(pubKeys, msgHashes); + if (allValid) { + for (size_t i = 0; i < pubKeys.size(); i++) { + jobs[indexes[i]].doneCallback(true); + } + } else { + // one or more sigs were not valid, revert to per-sig verification + // TODO this could be improved if we would cache pairing results in some way as the previous aggregated verification already calculated all the pairings for the hashes + for (size_t i = 0; i < pubKeys.size(); i++) { + auto& job = jobs[indexes[i]]; + bool valid = job.sig.VerifyInsecure(job.pubKey, job.msgHash); + job.doneCallback(valid); + } + } + } + + std::unique_lock l(sigVerifyMutex); + sigVerifyBatchesInProgress--; + if (!sigVerifyQueue.empty()) { + PushSigVerifyBatch(); + } + }; + + auto batch = std::make_shared >(std::move(sigVerifyQueue)); + sigVerifyQueue.reserve(SIG_VERIFY_BATCH_SIZE); + + sigVerifyBatchesInProgress++; + workerPool.push(f, batch); +} diff --git a/src/bls/bls_worker.h b/src/bls/bls_worker.h new file mode 100644 index 000000000000..b1f27ba018a5 --- /dev/null +++ b/src/bls/bls_worker.h @@ -0,0 +1,205 @@ +// Copyright (c) 2018 The Dash Core developers +// Copyright (c) 2021 The PIVX Core developers +// Distributed under the MIT/X11 software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef PIVX_CRYPTO_BLS_WORKER_H +#define PIVX_CRYPTO_BLS_WORKER_H + +#include "bls.h" +#include "ctpl.h" + +#include +#include + +#include + +// Low level BLS/DKG stuff. All very compute intensive and optimized for parallelization +// The worker tries to parallelize as much as possible and utilizes a few properties of BLS aggregation to speed up things +// For example, public key vectors can be aggregated in parallel if they are split into batches and the batched aggregations are +// aggregated to a final public key. This utilizes that when aggregating keys (a+b+c+d) gives the same result as (a+b)+(c+d) +class CBLSWorker +{ +public: + typedef std::function SignDoneCallback; + typedef std::function SigVerifyDoneCallback; + typedef std::function CancelCond; + +private: + ctpl::thread_pool workerPool; + + static const int SIG_VERIFY_BATCH_SIZE = 8; + struct SigVerifyJob { + SigVerifyDoneCallback doneCallback; + CancelCond cancelCond; + CBLSSignature sig; + CBLSPublicKey pubKey; + uint256 msgHash; + SigVerifyJob(SigVerifyDoneCallback&& _doneCallback, CancelCond&& _cancelCond, const CBLSSignature& _sig, const CBLSPublicKey& _pubKey, const uint256& _msgHash) : + doneCallback(_doneCallback), + cancelCond(_cancelCond), + sig(_sig), + pubKey(_pubKey), + msgHash(_msgHash) + { + } + }; + + std::mutex sigVerifyMutex; + int sigVerifyBatchesInProgress{0}; + std::vector sigVerifyQueue; + +public: + CBLSWorker(); + ~CBLSWorker(); + + void Start(); + void Stop(); + + bool GenerateContributions(int threshold, const BLSIdVector& ids, BLSVerificationVectorPtr& vvecRet, BLSSecretKeyVector& skShares); + + // The following functions are all used to aggregate verification (public key) vectors + // Inputs are in the following form: + // [ + // [a1, b1, c1, d1], + // [a2, b2, c2, d2], + // [a3, b3, c3, d3], + // [a4, b4, c4, d4], + // ] + // The result is in the following form: + // [ a1+a2+a3+a4, b1+b2+b3+b4, c1+c2+c3+c4, d1+d2+d3+d4] + // Multiple things can be parallelized here. For example, all 4 entries in the result vector can be calculated in parallel + // Also, each individual vector can be split into multiple batches and aggregating the batches can also be paralellized. + void AsyncBuildQuorumVerificationVector(const std::vector& vvecs, + size_t start, size_t count, bool parallel, + std::function doneCallback); + std::future AsyncBuildQuorumVerificationVector(const std::vector& vvecs, + size_t start, size_t count, bool parallel); + BLSVerificationVectorPtr BuildQuorumVerificationVector(const std::vector& vvecs, + size_t start = 0, size_t count = 0, bool parallel = true); + + // The following functions are all used to aggregate single vectors + // Inputs are in the following form: + // [a, b, c, d], + // The result is simply a+b+c+d + // Aggregation is paralellized by splitting up the input vector into multiple batches and then aggregating the individual batch results + void AsyncAggregateSecretKeys(const BLSSecretKeyVector& secKeys, + size_t start, size_t count, bool parallel, + std::function doneCallback); + std::future AsyncAggregateSecretKeys(const BLSSecretKeyVector& secKeys, + size_t start, size_t count, bool parallel); + CBLSSecretKey AggregateSecretKeys(const BLSSecretKeyVector& secKeys, size_t start = 0, size_t count = 0, bool parallel = true); + + void AsyncAggregatePublicKeys(const BLSPublicKeyVector& pubKeys, + size_t start, size_t count, bool parallel, + std::function doneCallback); + std::future AsyncAggregatePublicKeys(const BLSPublicKeyVector& pubKeys, + size_t start, size_t count, bool parallel); + CBLSPublicKey AggregatePublicKeys(const BLSPublicKeyVector& pubKeys, size_t start = 0, size_t count = 0, bool parallel = true); + + void AsyncAggregateSigs(const BLSSignatureVector& sigs, + size_t start, size_t count, bool parallel, + std::function doneCallback); + std::future AsyncAggregateSigs(const BLSSignatureVector& sigs, + size_t start, size_t count, bool parallel); + CBLSSignature AggregateSigs(const BLSSignatureVector& sigs, size_t start = 0, size_t count = 0, bool parallel = true); + + + // Calculate public key share from public key vector and id. Not parallelized + CBLSPublicKey BuildPubKeyShare(const BLSVerificationVectorPtr& vvec, const CBLSId& id); + + // The following functions verify multiple verification vectors and contributions for the same id + // This is parallelized by performing batched verification. The verification vectors and the contributions of + // a batch are aggregated (in parallel, see AsyncBuildQuorumVerificationVector and AsyncBuildSecretKeyShare). The + // result per batch is a single aggregated verification vector and a single aggregated contribution, which are then + // verified with VerifyContributionShare. If verification of the aggregated inputs is successful, the whole batch + // is marked as valid. If the batch verification fails, the individual entries are verified in a non-aggregated manner + void AsyncVerifyContributionShares(const CBLSId& forId, const std::vector& vvecs, const BLSSecretKeyVector& skShares, + bool parallel, bool aggregated, std::function&)> doneCallback); + std::future > AsyncVerifyContributionShares(const CBLSId& forId, const std::vector& vvecs, const BLSSecretKeyVector& skShares, + bool parallel, bool aggregated); + std::vector VerifyContributionShares(const CBLSId& forId, const std::vector& vvecs, const BLSSecretKeyVector& skShares, + bool parallel = true, bool aggregated = true); + + std::future AsyncVerifyContributionShare(const CBLSId& forId, const BLSVerificationVectorPtr& vvec, const CBLSSecretKey& skContribution); + + // Non paralellized verification of a single contribution + bool VerifyContributionShare(const CBLSId& forId, const BLSVerificationVectorPtr& vvec, const CBLSSecretKey& skContribution); + + // Simple verification of vectors. Checks x.IsValid() for every entry and checks for duplicate entries + bool VerifyVerificationVector(const BLSVerificationVector& vvec, size_t start = 0, size_t count = 0); + bool VerifyVerificationVectors(const std::vector& vvecs, size_t start = 0, size_t count = 0); + bool VerifySecretKeyVector(const BLSSecretKeyVector& secKeys, size_t start = 0, size_t count = 0); + bool VerifySignatureVector(const BLSSignatureVector& sigs, size_t start = 0, size_t count = 0); + + // Internally batched signature signing and verification + void AsyncSign(const CBLSSecretKey& secKey, const uint256& msgHash, SignDoneCallback doneCallback); + std::future AsyncSign(const CBLSSecretKey& secKey, const uint256& msgHash); + void AsyncVerifySig(const CBLSSignature& sig, const CBLSPublicKey& pubKey, const uint256& msgHash, SigVerifyDoneCallback doneCallback, CancelCond cancelCond = [] { return false; }); + std::future AsyncVerifySig(const CBLSSignature& sig, const CBLSPublicKey& pubKey, const uint256& msgHash, CancelCond cancelCond = [] { return false; }); + bool IsAsyncVerifyInProgress(); + +private: + void PushSigVerifyBatch(); +}; + +// Builds and caches different things from CBLSWorker +// Cache keys are provided externally as computing hashes on BLS vectors is too expensive +// If multiple threads try to build the same thing at the same time, only one will actually build it +// and the other ones will wait for the result of the first caller +class CBLSWorkerCache +{ +private: + CBLSWorker& worker; + + std::mutex cacheCs; + std::map > vvecCache; + std::map > secretKeyShareCache; + std::map > publicKeyShareCache; + +public: + CBLSWorkerCache(CBLSWorker& _worker) : + worker(_worker) {} + + BLSVerificationVectorPtr BuildQuorumVerificationVector(const uint256& cacheKey, const std::vector& vvecs) + { + return GetOrBuild(cacheKey, vvecCache, [&]() { + return worker.BuildQuorumVerificationVector(vvecs); + }); + } + CBLSSecretKey AggregateSecretKeys(const uint256& cacheKey, const BLSSecretKeyVector& skShares) + { + return GetOrBuild(cacheKey, secretKeyShareCache, [&]() { + return worker.AggregateSecretKeys(skShares); + }); + } + CBLSPublicKey BuildPubKeyShare(const uint256& cacheKey, const BLSVerificationVectorPtr& vvec, const CBLSId& id) + { + return GetOrBuild(cacheKey, publicKeyShareCache, [&]() { + return worker.BuildPubKeyShare(vvec, id); + }); + } + +private: + template + T GetOrBuild(const uint256& cacheKey, std::map >& cache, Builder&& builder) + { + cacheCs.lock(); + auto it = cache.find(cacheKey); + if (it != cache.end()) { + auto f = it->second; + cacheCs.unlock(); + return f.get(); + } + + std::promise p; + cache.emplace(cacheKey, p.get_future()); + cacheCs.unlock(); + + T v = builder(); + p.set_value(v); + return v; + } +}; + +#endif // PIVX_CRYPTO_BLS_WORKER_H diff --git a/src/ctpl.h b/src/ctpl.h new file mode 100644 index 000000000000..b98dfddd8523 --- /dev/null +++ b/src/ctpl.h @@ -0,0 +1,239 @@ + +/********************************************************* + * + * Copyright (C) 2014 by Vitaliy Vitsentiy + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *********************************************************/ + + +#ifndef __ctpl_thread_pool_H__ +#define __ctpl_thread_pool_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#ifndef _ctplThreadPoolLength_ +#define _ctplThreadPoolLength_ 100 +#endif + + +// thread pool to run user's functors with signature +// ret func(int id, other_params) +// where id is the index of the thread that runs the functor +// ret is some return type + + +namespace ctpl { + + class thread_pool { + + public: + + thread_pool() : q(_ctplThreadPoolLength_) { this->init(); } + thread_pool(int nThreads, int queueSize = _ctplThreadPoolLength_) : q(queueSize) { this->init(); this->resize(nThreads); } + + // the destructor waits for all the functions in the queue to be finished + ~thread_pool() { + this->stop(true); + } + + // get the number of running threads in the pool + int size() { return static_cast(this->threads.size()); } + + // number of idle threads + int n_idle() { return this->nWaiting; } + std::thread & get_thread(int i) { return *this->threads[i]; } + + // change the number of threads in the pool + // should be called from one thread, otherwise be careful to not interleave, also with this->stop() + // nThreads must be >= 0 + void resize(int nThreads) { + if (!this->isStop && !this->isDone) { + int oldNThreads = static_cast(this->threads.size()); + if (oldNThreads <= nThreads) { // if the number of threads is increased + this->threads.resize(nThreads); + this->flags.resize(nThreads); + + for (int i = oldNThreads; i < nThreads; ++i) { + this->flags[i] = std::make_shared>(false); + this->set_thread(i); + } + } + else { // the number of threads is decreased + for (int i = oldNThreads - 1; i >= nThreads; --i) { + *this->flags[i] = true; // this thread will finish + this->threads[i]->detach(); + } + { + // stop the detached threads that were waiting + std::unique_lock lock(this->mutex); + this->cv.notify_all(); + } + this->threads.resize(nThreads); // safe to delete because the threads are detached + this->flags.resize(nThreads); // safe to delete because the threads have copies of shared_ptr of the flags, not originals + } + } + } + + // empty the queue + void clear_queue() { + std::function * _f; + while (this->q.pop(_f)) + delete _f; // empty the queue + } + + // pops a functional wraper to the original function + std::function pop() { + std::function * _f = nullptr; + this->q.pop(_f); + std::unique_ptr> func(_f); // at return, delete the function even if an exception occurred + + std::function f; + if (_f) + f = *_f; + return f; + } + + + // wait for all computing threads to finish and stop all threads + // may be called asyncronously to not pause the calling thread while waiting + // if isWait == true, all the functions in the queue are run, otherwise the queue is cleared without running the functions + void stop(bool isWait = false) { + if (!isWait) { + if (this->isStop) + return; + this->isStop = true; + for (int i = 0, n = this->size(); i < n; ++i) { + *this->flags[i] = true; // command the threads to stop + } + this->clear_queue(); // empty the queue + } + else { + if (this->isDone || this->isStop) + return; + this->isDone = true; // give the waiting threads a command to finish + } + { + std::unique_lock lock(this->mutex); + this->cv.notify_all(); // stop all waiting threads + } + for (int i = 0; i < static_cast(this->threads.size()); ++i) { // wait for the computing threads to finish + if (this->threads[i]->joinable()) + this->threads[i]->join(); + } + // if there were no threads in the pool but some functors in the queue, the functors are not deleted by the threads + // therefore delete them here + this->clear_queue(); + this->threads.clear(); + this->flags.clear(); + } + + template + auto push(F && f, Rest&&... rest) ->std::future { + auto pck = std::make_shared>( + std::bind(std::forward(f), std::placeholders::_1, std::forward(rest)...) + ); + + auto _f = new std::function([pck](int id) { + (*pck)(id); + }); + this->q.push(_f); + + std::unique_lock lock(this->mutex); + this->cv.notify_one(); + + return pck->get_future(); + } + + // run the user's function that excepts argument int - id of the running thread. returned value is templatized + // operator returns std::future, where the user can get the result and rethrow the catched exceptins + template + auto push(F && f) ->std::future { + auto pck = std::make_shared>(std::forward(f)); + + auto _f = new std::function([pck](int id) { + (*pck)(id); + }); + this->q.push(_f); + + std::unique_lock lock(this->mutex); + this->cv.notify_one(); + + return pck->get_future(); + } + + + private: + + // deleted + thread_pool(const thread_pool &);// = delete; + thread_pool(thread_pool &&);// = delete; + thread_pool & operator=(const thread_pool &);// = delete; + thread_pool & operator=(thread_pool &&);// = delete; + + void set_thread(int i) { + std::shared_ptr> flag(this->flags[i]); // a copy of the shared ptr to the flag + auto f = [this, i, flag/* a copy of the shared ptr to the flag */]() { + std::atomic & _flag = *flag; + std::function * _f; + bool isPop = this->q.pop(_f); + while (true) { + while (isPop) { // if there is anything in the queue + std::unique_ptr> func(_f); // at return, delete the function even if an exception occurred + (*_f)(i); + + if (_flag) + return; // the thread is wanted to stop, return even if the queue is not empty yet + else + isPop = this->q.pop(_f); + } + + // the queue is empty here, wait for the next command + std::unique_lock lock(this->mutex); + ++this->nWaiting; + this->cv.wait(lock, [this, &_f, &isPop, &_flag](){ isPop = this->q.pop(_f); return isPop || this->isDone || _flag; }); + --this->nWaiting; + + if (!isPop) + return; // if the queue is empty and this->isDone == true or *flag then return + } + }; + this->threads[i].reset(new std::thread(f)); // compiler may not support std::make_unique() + } + + void init() { this->nWaiting = 0; this->isStop = false; this->isDone = false; } + + std::vector> threads; + std::vector>> flags; + mutable boost::lockfree::queue *> q; + std::atomic isDone; + std::atomic isStop; + std::atomic nWaiting; // how many threads are waiting + + std::mutex mutex; + std::condition_variable cv; + }; + +} + +#endif // __ctpl_thread_pool_H__ \ No newline at end of file diff --git a/src/init.cpp b/src/init.cpp index f00b2fc4d59f..8873a29887c3 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -17,6 +17,7 @@ #include "activemasternode.h" #include "addrman.h" #include "amount.h" +#include "bls/bls.h" #include "budget/budgetdb.h" #include "budget/budgetmanager.h" #include "checkpoints.h" @@ -763,14 +764,19 @@ bool InitSanityCheck(void) return false; } - if (!glibc_sanity_test() || !glibcxx_sanity_test()) + if (!glibc_sanity_test() || !glibcxx_sanity_test()) { return false; + } if (!Random_SanityCheck()) { UIError(_("OS cryptographic RNG sanity check failure. Aborting.")); return false; } + if (!BLSInit()) { + return false; + } + return true; } diff --git a/src/logging.cpp b/src/logging.cpp index af96e9edb207..4eed10e5b4b4 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -102,7 +102,7 @@ const CLogCategoryDesc LogCategories[] = { {BCLog::TOR, "tor"}, {BCLog::MEMPOOL, "mempool"}, {BCLog::HTTP, "http"}, - {BCLog::BENCH, "bench"}, + {BCLog::BENCHMARK, "bench"}, {BCLog::ZMQ, "zmq"}, {BCLog::DB, "db"}, {BCLog::RPC, "rpc"}, @@ -111,7 +111,7 @@ const CLogCategoryDesc LogCategories[] = { {BCLog::SELECTCOINS, "selectcoins"}, {BCLog::REINDEX, "reindex"}, {BCLog::CMPCTBLOCK, "cmpctblock"}, - {BCLog::RAND, "rand"}, + {BCLog::RANDOM, "rand"}, {BCLog::PRUNE, "prune"}, {BCLog::PROXY, "proxy"}, {BCLog::MEMPOOLREJ, "mempoolrej"}, diff --git a/src/logging.h b/src/logging.h index a433acd327f3..e17b32db2ca7 100644 --- a/src/logging.h +++ b/src/logging.h @@ -41,7 +41,7 @@ namespace BCLog { TOR = (1 << 1), MEMPOOL = (1 << 2), HTTP = (1 << 3), - BENCH = (1 << 4), + BENCHMARK = (1 << 4), ZMQ = (1 << 5), DB = (1 << 6), RPC = (1 << 7), @@ -50,7 +50,7 @@ namespace BCLog { SELECTCOINS = (1 << 10), REINDEX = (1 << 11), CMPCTBLOCK = (1 << 12), - RAND = (1 << 13), + RANDOM = (1 << 13), PRUNE = (1 << 14), PROXY = (1 << 15), MEMPOOLREJ = (1 << 16), diff --git a/src/support/allocators/mt_pooled_secure.h b/src/support/allocators/mt_pooled_secure.h new file mode 100644 index 000000000000..56f6cdc2f2ec --- /dev/null +++ b/src/support/allocators/mt_pooled_secure.h @@ -0,0 +1,86 @@ +// Copyright (c) 2018-2021 The Dash Core developers +// Copyright (c) 2021 The PIVX developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef PIVX_SUPPORT_ALLOCATORS_MT_POOLED_SECURE_H +#define PIVX_SUPPORT_ALLOCATORS_MT_POOLED_SECURE_H + +#include "pooled_secure.h" + +#include +#include + +// +// Manages a pool of pools to balance allocation between those when multiple threads are involved +// This allocator is fully thread safe +// +template +struct mt_pooled_secure_allocator : public std::allocator { + // MSVC8 default copy constructor is broken + typedef std::allocator base; + typedef typename base::size_type size_type; + typedef typename base::difference_type difference_type; + typedef typename base::pointer pointer; + typedef typename base::const_pointer const_pointer; + typedef typename base::reference reference; + typedef typename base::const_reference const_reference; + typedef typename base::value_type value_type; + mt_pooled_secure_allocator(size_type nrequested_size = 32, + size_type nnext_size = 32, + size_type nmax_size = 0) throw() + { + // we add enough bytes to the requested size so that we can store the bucket as well + nrequested_size += sizeof(size_t); + + size_t pools_count = std::thread::hardware_concurrency(); + pools.resize(pools_count); + for (size_t i = 0; i < pools_count; i++) { + pools[i] = std::make_unique(nrequested_size, nnext_size, nmax_size); + } + } + ~mt_pooled_secure_allocator() throw() {} + + T* allocate(std::size_t n, const void* hint = 0) + { + size_t bucket = get_bucket(); + std::lock_guard lock(pools[bucket]->mutex); + uint8_t* ptr = pools[bucket]->allocate(n * sizeof(T) + sizeof(size_t)); + *(size_t*)ptr = bucket; + return static_cast(ptr + sizeof(size_t)); + } + + void deallocate(T* p, std::size_t n) + { + if (!p) { + return; + } + uint8_t* ptr = (uint8_t*)p - sizeof(size_t); + size_t bucket = *(size_t*)ptr; + std::lock_guard lock(pools[bucket]->mutex); + pools[bucket]->deallocate(ptr, n * sizeof(T)); + } + +private: + size_t get_bucket() + { + auto tid = std::this_thread::get_id(); + size_t x = std::hash{}(std::this_thread::get_id()); + return x % pools.size(); + } + + struct internal_pool : pooled_secure_allocator { + internal_pool(size_type nrequested_size, + size_type nnext_size, + size_type nmax_size) : + pooled_secure_allocator(nrequested_size, nnext_size, nmax_size) + { + } + std::mutex mutex; + }; + +private: + std::vector> pools; +}; + +#endif // PIVX_SUPPORT_ALLOCATORS_MT_POOLED_SECURE_H diff --git a/src/support/allocators/pooled_secure.h b/src/support/allocators/pooled_secure.h new file mode 100644 index 000000000000..de2057e76a35 --- /dev/null +++ b/src/support/allocators/pooled_secure.h @@ -0,0 +1,74 @@ +// Copyright (c) 2018-2021 The Dash Core developers +// Copyright (c) 2021 The PIVX developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef PIVX_SUPPORT_ALLOCATORS_POOLED_SECURE_H +#define PIVX_SUPPORT_ALLOCATORS_POOLED_SECURE_H + +#include "support/lockedpool.h" +#include "support/cleanse.h" + +#include +#include + +#include + +// +// Allocator that allocates memory in chunks from a pool, which in turn allocates larger chunks from secure memory +// Memory is cleaned when freed as well. This allocator is NOT thread safe +// +template +struct pooled_secure_allocator : public std::allocator { + // MSVC8 default copy constructor is broken + typedef std::allocator base; + typedef typename base::size_type size_type; + typedef typename base::difference_type difference_type; + typedef typename base::pointer pointer; + typedef typename base::const_pointer const_pointer; + typedef typename base::reference reference; + typedef typename base::const_reference const_reference; + typedef typename base::value_type value_type; + pooled_secure_allocator(const size_type nrequested_size = 32, + const size_type nnext_size = 32, + const size_type nmax_size = 0) throw() : + pool(nrequested_size, nnext_size, nmax_size){} + ~pooled_secure_allocator() throw() {} + + T* allocate(std::size_t n, const void* hint = 0) + { + size_t chunks = (n * sizeof(T) + pool.get_requested_size() - 1) / pool.get_requested_size(); + return static_cast(pool.ordered_malloc(chunks)); + } + + void deallocate(T* p, std::size_t n) + { + if (!p) { + return; + } + + size_t chunks = (n * sizeof(T) + pool.get_requested_size() - 1) / pool.get_requested_size(); + memory_cleanse(p, chunks * pool.get_requested_size()); + pool.ordered_free(p, chunks); + } + +public: + struct internal_secure_allocator { + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + + static char* malloc(const size_type bytes) + { + return static_cast(LockedPoolManager::Instance().alloc(bytes)); + } + + static void free(char* const block) + { + LockedPoolManager::Instance().free(block); + } + }; +private: + boost::pool pool; +}; + +#endif // PIVX_SUPPORT_ALLOCATORS_POOLED_SECURE_H diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index db5df51ce43f..00f35727b32e 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -86,6 +86,7 @@ set(BITCOIN_TESTS ${CMAKE_CURRENT_SOURCE_DIR}/bech32_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/budget_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/bip32_tests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/bls_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/checkblock_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Checkpoints_tests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/coins_tests.cpp diff --git a/src/test/bls_tests.cpp b/src/test/bls_tests.cpp new file mode 100644 index 000000000000..5e2c27127b37 --- /dev/null +++ b/src/test/bls_tests.cpp @@ -0,0 +1,54 @@ +// Copyright (c) 2019-2020 The Dash Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "test/test_pivx.h" +#include "bls/bls.h" + +#include + +BOOST_FIXTURE_TEST_SUITE(bls_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(bls_sethexstr_tests) +{ + CBLSSecretKey sk; + std::string strValidSecret = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"; + // Note: invalid string passed to SetHexStr() should cause it to fail and reset key internal data + BOOST_CHECK(sk.SetHexStr(strValidSecret)); + BOOST_CHECK(!sk.SetHexStr("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1g")); // non-hex + BOOST_CHECK(!sk.IsValid()); + BOOST_CHECK(sk == CBLSSecretKey()); + // Try few more invalid strings + BOOST_CHECK(sk.SetHexStr(strValidSecret)); + BOOST_CHECK(!sk.SetHexStr("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e")); // hex but too short + BOOST_CHECK(!sk.IsValid()); + BOOST_CHECK(sk.SetHexStr(strValidSecret)); + BOOST_CHECK(!sk.SetHexStr("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20")); // hex but too long + BOOST_CHECK(!sk.IsValid()); +} + +BOOST_AUTO_TEST_CASE(bls_sig_tests) +{ + CBLSSecretKey sk1, sk2; + sk1.MakeNewKey(); + sk2.MakeNewKey(); + + uint256 msgHash1 = uint256S("0000000000000000000000000000000000000000000000000000000000000001"); + uint256 msgHash2 = uint256S("0000000000000000000000000000000000000000000000000000000000000002"); + + auto sig1 = sk1.Sign(msgHash1); + auto sig2 = sk2.Sign(msgHash1); + + BOOST_CHECK(sig1.VerifyInsecure(sk1.GetPublicKey(), msgHash1)); + BOOST_CHECK(!sig1.VerifyInsecure(sk1.GetPublicKey(), msgHash2)); + + BOOST_CHECK(sig2.VerifyInsecure(sk2.GetPublicKey(), msgHash1)); + BOOST_CHECK(!sig2.VerifyInsecure(sk2.GetPublicKey(), msgHash2)); + + BOOST_CHECK(!sig1.VerifyInsecure(sk2.GetPublicKey(), msgHash1)); + BOOST_CHECK(!sig1.VerifyInsecure(sk2.GetPublicKey(), msgHash2)); + BOOST_CHECK(!sig2.VerifyInsecure(sk1.GetPublicKey(), msgHash1)); + BOOST_CHECK(!sig2.VerifyInsecure(sk1.GetPublicKey(), msgHash2)); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/src/test/reverselock_tests.cpp b/src/test/reverselock_tests.cpp index 6777a5298474..622f3b98b8de 100644 --- a/src/test/reverselock_tests.cpp +++ b/src/test/reverselock_tests.cpp @@ -3,6 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include "test/test_pivx.h" #include "reverselock.h" #include diff --git a/src/test/test_pivx.cpp b/src/test/test_pivx.cpp index 46f811cdffcb..3ba623cb68b7 100644 --- a/src/test/test_pivx.cpp +++ b/src/test/test_pivx.cpp @@ -8,6 +8,7 @@ #include "test/test_pivx.h" #include "blockassembler.h" +#include "bls/bls.h" #include "guiinterface.h" #include "evo/deterministicmns.h" #include "evo/evodb.h" @@ -42,6 +43,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName) : m_path_root(fs::temp_directory_path() / "test_pivx" / strprintf("%lu_%i", (unsigned long)GetTime(), (int)(InsecureRandRange(1 << 30)))) { ECC_Start(); + BLSInit(); SetupEnvironment(); InitSignatureCache(); fCheckBlockIndex = true; diff --git a/src/util/threadnames.cpp b/src/util/threadnames.cpp index e5402d8d87f8..f455be5d3e7a 100644 --- a/src/util/threadnames.cpp +++ b/src/util/threadnames.cpp @@ -15,6 +15,10 @@ #include +#include "ctpl.h" +#include "utiltime.h" +#include "tinyformat.h" + #ifdef HAVE_SYS_PRCTL_H #include // For prctl, PR_SET_NAME, PR_GET_NAME #endif @@ -64,3 +68,22 @@ void util::ThreadSetInternalName(std::string&& name) { SetInternalName(std::move(name)); } + +void RenameThreadPool(ctpl::thread_pool& tp, const char* baseName) +{ + auto cond = std::make_shared(); + auto mutex = std::make_shared(); + std::atomic doneCnt(0); + for (int i = 0; i < tp.size(); i++) { + tp.push([baseName, i, cond, mutex, &doneCnt](int threadId) { + util::ThreadRename(strprintf("%s-%d", baseName, i).c_str()); + doneCnt++; + std::unique_lock l(*mutex); + cond->wait(l); + }); + } + while (doneCnt != tp.size()) { + MilliSleep(10); + } + cond->notify_all(); +} diff --git a/src/util/threadnames.h b/src/util/threadnames.h index 64b2689cf137..e2c603edacd4 100644 --- a/src/util/threadnames.h +++ b/src/util/threadnames.h @@ -23,4 +23,9 @@ const std::string& ThreadGetInternalName(); } // namespace util +namespace ctpl { + class thread_pool; +} +void RenameThreadPool(ctpl::thread_pool& tp, const char* baseName); + #endif // BITCOIN_UTIL_THREADNAMES_H diff --git a/src/validation.cpp b/src/validation.cpp index 8784e976ad14..c5b7b7110271 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1702,7 +1702,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd int64_t nTime1 = GetTimeMicros(); nTimeConnect += nTime1 - nTimeStart; - LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime1 - nTimeStart), 0.001 * (nTime1 - nTimeStart) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime1 - nTimeStart) / (nInputs - 1), nTimeConnect * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime1 - nTimeStart), 0.001 * (nTime1 - nTimeStart) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime1 - nTimeStart) / (nInputs - 1), nTimeConnect * 0.000001); //PoW phase redistributed fees to miner. PoS stage destroys fees. CAmount nExpectedMint = GetBlockValue(pindex->nHeight); @@ -1727,14 +1727,14 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd return state.DoS(100, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed"); int64_t nTime2 = GetTimeMicros(); nTimeVerify += nTime2 - nTimeStart; - LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime2 - nTimeStart), nInputs <= 1 ? 0 : 0.001 * (nTime2 - nTimeStart) / (nInputs - 1), nTimeVerify * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime2 - nTimeStart), nInputs <= 1 ? 0 : 0.001 * (nTime2 - nTimeStart) / (nInputs - 1), nTimeVerify * 0.000001); if (!ProcessSpecialTxsInBlock(block, pindex, state, fJustCheck)) { return error("%s: Special tx processing failed with %s", __func__, FormatStateMessage(state)); } int64_t nTime3 = GetTimeMicros(); nTimeProcessSpecial += nTime3 - nTime2; - LogPrint(BCLog::BENCH, " - Process special tx: %.2fms [%.2fs]\n", 0.001 * (nTime3 - nTime2), nTimeProcessSpecial * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Process special tx: %.2fms [%.2fs]\n", 0.001 * (nTime3 - nTime2), nTimeProcessSpecial * 0.000001); //IMPORTANT NOTE: Nothing before this point should actually store to disk (or even memory) if (fJustCheck) @@ -1772,7 +1772,7 @@ static bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockInd int64_t nTime4 = GetTimeMicros(); nTimeIndex += nTime4 - nTime3; - LogPrint(BCLog::BENCH, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime4 - nTime3), nTimeIndex * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime4 - nTime3), nTimeIndex * 0.000001); if (consensus.NetworkUpgradeActive(pindex->nHeight, Consensus::UPGRADE_ZC_V2) && pindex->nHeight < consensus.height_last_ZC_AccumCheckpoint) { @@ -1985,7 +1985,7 @@ bool static DisconnectTip(CValidationState& state, const CChainParams& chainpara assert(flushed); dbTx->Commit(); } - LogPrint(BCLog::BENCH, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); + LogPrint(BCLog::BENCHMARK, "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); const uint256& saplingAnchorAfterDisconnect = pcoinsTip->GetBestAnchor(); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) @@ -2100,7 +2100,7 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const st int64_t nTime2 = GetTimeMicros(); nTimeReadFromDisk += nTime2 - nTime1; int64_t nTime3; - LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { auto dbTx = evoDb->BeginTransaction(); @@ -2114,14 +2114,14 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const st } nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; - LogPrint(BCLog::BENCH, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); bool flushed = view.Flush(); assert(flushed); dbTx->Commit(); } int64_t nTime4 = GetTimeMicros(); nTimeFlush += nTime4 - nTime3; - LogPrint(BCLog::BENCH, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Flush: %.2fms [%.2fs]\n", (nTime4 - nTime3) * 0.001, nTimeFlush * 0.000001); // Write the chain state to disk, if necessary. Always write to disk if this is the first of a new file. FlushStateMode flushMode = FLUSH_STATE_IF_NEEDED; @@ -2131,7 +2131,7 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const st return false; int64_t nTime5 = GetTimeMicros(); nTimeChainState += nTime5 - nTime4; - LogPrint(BCLog::BENCH, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool. mempool.removeForBlock(blockConnecting.vtx, pindexNew->nHeight, !IsInitialBlockDownload()); @@ -2145,8 +2145,8 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, const st int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; - LogPrint(BCLog::BENCH, " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); - LogPrint(BCLog::BENCH, "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); + LogPrint(BCLog::BENCHMARK, " - Connect postprocess: %.2fms [%.2fs]\n", (nTime6 - nTime5) * 0.001, nTimePostConnect * 0.000001); + LogPrint(BCLog::BENCHMARK, "- Connect block: %.2fms [%.2fs]\n", (nTime6 - nTime1) * 0.001, nTimeTotal * 0.000001); connectTrace.BlockConnected(pindexNew, std::move(pthisBlock)); return true;